diff --git a/.github/actions/diskspace-clean/action.yml b/.github/actions/diskspace-clean/action.yml new file mode 100644 index 000000000..b50ee5278 --- /dev/null +++ b/.github/actions/diskspace-clean/action.yml @@ -0,0 +1,23 @@ +name: Cleans up diskspace + +description: Cleans up diskspace + +runs: + using: composite + steps: + - name: Cleans up diskspace + shell: bash + run: | + set -xe -o pipefail + # Clean workspace + rm -rf ${{ github.workspace }}/* || sudo rm -rf ${{ github.workspace }}/* + # Clean cache + rm -rf /tmp/ || sudo rm -rf /tmp/ + mkdir -m 777 /tmp || sudo mkdir -m 777 /tmp + rm -rf ~/.torch || sudo rm -rf ~/.torch + rm -rf ~/.triton || sudo rm -rf ~/.triton + rm -rf ~/.cache || sudo rm -rf ~/.cache + rm -rf ~/.conda || sudo rm -rf ~/.conda + # Clean docker + docker stop $(docker ps -aq) || true + docker system prune -af diff --git a/.github/actions/inductor-xpu-e2e-test/action.yml b/.github/actions/inductor-xpu-e2e-test/action.yml index a1c457aa6..92baad183 100644 --- a/.github/actions/inductor-xpu-e2e-test/action.yml +++ b/.github/actions/inductor-xpu-e2e-test/action.yml @@ -52,14 +52,6 @@ runs: run: | source activate e2e_ci if [[ ${{ inputs.suite }} == *"torchbench"* ]]; then - if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then - cd ../ && rm -rf audio && git clone --single-branch -b main https://github.com/pytorch/audio.git - cd audio && git checkout $TORCHAUDIO_COMMIT_ID - python setup.py bdist_wheel && pip uninstall torchaudio -y && pip install dist/*.whl - cd ../ && rm -rf vision && git clone --single-branch -b main https://github.com/pytorch/vision.git - cd vision && git checkout $TORCHVISION_COMMIT_ID - python setup.py bdist_wheel && pip uninstall torchvision -y && pip install dist/*.whl - fi cd ../ && python -c "import torch, torchvision, torchaudio" rm -rf benchmark && git clone https://github.com/pytorch/benchmark.git cd benchmark && git checkout $TORCHBENCH_COMMIT_ID @@ -80,11 +72,6 @@ runs: pip install --force-reinstall git+https://github.com/huggingface/transformers@${TRANSFORMERS_VERSION} fi if [[ ${{ inputs.suite }} == *"timm_models"* ]]; then - if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then - cd ../ && rm -rf vision && git clone --single-branch -b main https://github.com/pytorch/vision.git - cd vision && git checkout $TORCHVISION_COMMIT_ID - python setup.py bdist_wheel && pip uninstall torchvision -y && pip install dist/*.whl - fi # install timm without dependencies pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID # install timm dependencies without torch and torchvision diff --git a/.github/actions/testenv-setup/action.yml b/.github/actions/testenv-setup/action.yml new file mode 100644 index 000000000..ded6af88d --- /dev/null +++ b/.github/actions/testenv-setup/action.yml @@ -0,0 +1,57 @@ +name: Cleans up diskspace + +description: Cleans up diskspace + +inputs: + suite: + required: true + type: string + default: 'huggingface' + description: Dynamo benchmarks test suite. huggingface,timm_models,torchbench. Delimiter is comma + env_prepare: + required: false + description: If set to any value, will prepare suite test env + dt: + required: true + type: string + default: 'float32' + description: Data precision of the test.float32,bfloat16,float16,amp_bf16,amp_fp16. Delimiter is comma + mode: + required: true + type: string + default: 'inference' + description: inference,training. Delimiter is comma + scenario: + required: true + type: string + default: 'accuracy' + description: accuracy,performance. Delimiter is comma + cards: + required: false + type: string + default: 'all' + description: which cards can be used in the test + hf_token: + required: false + description: HUGGING_FACE_HUB_TOKEN for torchbench test + pytorch: + required: false + type: string + default: 'main' + description: Pytorch branch/commit + driver: + required: false + type: string + default: 'lts' + description: Driver lts/rolling + +runs: + using: composite + defaults: + run: + shell: bash -xe -o pipefail + steps: + - name: Prepare PyTorch + shell: bash + run: | + echo diff --git a/.github/scripts/build.sh b/.github/scripts/build.sh index 9dcd170aa..485f2cae2 100755 --- a/.github/scripts/build.sh +++ b/.github/scripts/build.sh @@ -44,7 +44,7 @@ cp -r ${WORKSPACE}/torch-xpu-ops third_party/torch-xpu-ops # Pre Build cd ${WORKSPACE}/pytorch -python -m pip install requests +python -m pip install requests cmake ninja python third_party/torch-xpu-ops/.github/scripts/apply_torch_pr.py git submodule sync && git submodule update --init --recursive python -m pip install -r requirements.txt @@ -85,6 +85,17 @@ rm -rf ./tmp bash third_party/torch-xpu-ops/.github/scripts/rpath.sh ${WORKSPACE}/pytorch/dist/torch*.whl python -m pip install --force-reinstall tmp/torch*.whl +# Build torchvision torchaudio +unset PYTORCH_VERSION +TORCHVISION_COMMIT=$(cat .github/ci_commit_pins/vision.txt) +rm -rf xpu-vision && git clone https://github.com/pytorch/vision.git xpu-vision +cd xpu-vision && git checkout ${TORCHVISION_COMMIT} +python setup.py bdist_wheel && cd .. +TORCHAUDIO_COMMIT=$(cat .github/ci_commit_pins/audio.txt) +rm -rf xpu-audio && git clone https://github.com/pytorch/audio.git xpu-audio +cd xpu-audio && git checkout ${TORCHAUDIO_COMMIT} +python setup.py bdist_wheel && cd .. + # Verify cd ${WORKSPACE} python ${WORKSPACE}/pytorch/torch/utils/collect_env.py @@ -95,6 +106,8 @@ xpu_is_compiled="$(python -c 'import torch; print(torch.xpu._is_compiled())')" # Save wheel if [ "${xpu_is_compiled,,}" == "true" ];then cp ${WORKSPACE}/pytorch/tmp/torch*.whl ${WORKSPACE} + cp ${WORKSPACE}/pytorch/xpu-vision/dist/torchvision*.whl ${WORKSPACE} + cp ${WORKSPACE}/pytorch/xpu-audio/dist/torchaudio*.whl ${WORKSPACE} else echo "Build got failed!" exit 1 diff --git a/.github/scripts/lintrunner.sh b/.github/scripts/lintrunner.sh index 6d3122b6f..099ee74d4 100755 --- a/.github/scripts/lintrunner.sh +++ b/.github/scripts/lintrunner.sh @@ -24,7 +24,7 @@ if ! command -v lintrunner &> /dev/null; then fi # Ignoring errors in one specific run -export SHELLCHECK_OPTS="-e SC2154 -e SC2086 -e SC1091 -e SC2046" +export SHELLCHECK_OPTS="-e SC2154 -e SC2086 -e SC1091 -e SC2046 -e SC1090" # This has already been cached in the docker image lintrunner init 2> /dev/null diff --git a/.github/scripts/setup_test_env.sh b/.github/scripts/setup_test_env.sh new file mode 100755 index 000000000..d27e2aa78 --- /dev/null +++ b/.github/scripts/setup_test_env.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -xe -o pipefail +export GIT_PAGER=cat + +# Init params +WORKSPACE=$(realpath ${WORKSPACE:-"/tmp"}) +CONDA_ENV=${CONDA_ENV:-"xpu-test"} +PYTHON_VERSION=${PYTHON_VERSION:-"3.10"} +PYTORCH_REPO=${PYTORCH_REPO:-"https://github.com/pytorch/pytorch.git"} +PYTORCH_VERSION=${PYTORCH_VERSION:-"main"} +for var; do + eval "export $(echo ${var@Q} |sed "s/^'-*//g;s/=/='/")" +done + +# Python env via conda +. "$(conda info -e |awk '{if($1=="base"){printf("%s/etc/profile.d/conda.sh", $NF)}}')" +conda create python=${PYTHON_VERSION} -y -n ${CONDA_ENV} +conda activate ${CONDA_ENV} +conda info -e +which python && python -V && conda list +python -m pip install requests pandas scipy psutil + +# Prepare pytorch +if [ "${PYTORCH_VERSION}" == "release" ];then + python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/xpu +elif [ "${PYTORCH_VERSION}" == "test" ];then + python -m pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/test/xpu +elif [ "${PYTORCH_VERSION}" == "nightly" ];then + python -m pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu +else + python -m pip install ${WORKSPACE}/torch*.whl +fi +TORCH_COMMIT="$(python -c 'import torch; print(torch.version.git_version)')" +rm -rf ./pytorch +git clone ${PYTORCH_REPO} pytorch +cd pytorch +git checkout ${TORCH_COMMIT} +git remote -v && git branch && git show -s + +# Prepare torch-xpu-ops +rm -rf third_party/torch-xpu-ops +if [ "${PYTORCH_VERSION}" != "main" ];then + TORCH_XPU_OPS_COMMIT=$(cat third_party/xpu.txt) + git clone https://github.com/intel/torch-xpu-ops.git third_party/torch-xpu-ops + cd third_party/torch-xpu-ops + git checkout ${TORCH_XPU_OPS_COMMIT} +else + cp -r ${WORKSPACE} third_party/torch-xpu-ops + cd third_party/torch-xpu-ops +fi +git remote -v && git branch && git show -s +cd ../.. +if [ "${GITHUB_EVENT_NAME}" == "pull_request" ];then + python third_party/torch-xpu-ops/.github/scripts/apply_torch_pr.py -e https://github.com/pytorch/pytorch/pull/152940 +else + python third_party/torch-xpu-ops/.github/scripts/apply_torch_pr.py +fi + +# Install triton +if [ "${TRITON_VERSION}" == "pinned" ];then + TRITON_VERSION="$(cat .ci/docker/ci_commit_pins/triton-xpu.txt)" +fi +if [ -n "${TRITON_VERSION}" ];then + TRITON_REPO="https://github.com/intel/intel-xpu-backend-for-triton" + python -m pip uninstall -y pytorch-triton-xpu + python -m pip install "git+${TRITON_REPO}@${TRITON_VERSION}#subdirectory=python" +fi + +# Install requirements +python -m pip install -r .ci/docker/requirements-ci.txt +python -m pip install -U pytest pytest-timeout + +# Collect env infos +cd .. +python -c "import torch; print(torch.__config__.show())" +python -c "import torch; print(torch.__config__.parallel_info())" +python -c "import torch; print(torch.__config__.torch.xpu.device_count())" +python -c "import triton; print(triton.__version__)" +python pytorch/torch/utils/collect_env.py + +# Clean cache +rm -rf /tmp/ || sudo rm -rf /tmp/ +mkdir -m 777 /tmp || sudo mkdir -m 777 /tmp +rm -rf ~/.torch || sudo rm -rf ~/.torch +rm -rf ~/.triton || sudo rm -rf ~/.triton +rm -rf ~/.cache || sudo rm -rf ~/.cache diff --git a/.github/workflows/_linux_build.yml b/.github/workflows/_linux_build.yml index 3749e7992..fe73a98fd 100644 --- a/.github/workflows/_linux_build.yml +++ b/.github/workflows/_linux_build.yml @@ -28,11 +28,6 @@ on: type: string default: 'linux.idc.xpu' description: Runner label - update_lkg: - required: false - type: string - default: 'false' - description: Whether update LKG torch version to issue #1280 outputs: torch_commit_id: description: The commit id of the torch build diff --git a/.github/workflows/_linux_ut.yml b/.github/workflows/_linux_ut.yml index 81f849e6f..13ac01d69 100644 --- a/.github/workflows/_linux_ut.yml +++ b/.github/workflows/_linux_ut.yml @@ -48,95 +48,37 @@ jobs: ut_test: runs-on: ${{ inputs.runner }} if: ${{ inputs.ut != 'xpu_distributed' && !contains(inputs.disabled_tests, 'disable_ut') }} - timeout-minutes: 300 + timeout-minutes: 600 env: GH_TOKEN: ${{ github.token }} NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }} DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }} ut_skip_issue: 1624 steps: - - name: Checkout torch-xpu-ops + - name: Checkout Torch-xpu-ops uses: actions/checkout@v4 - - name: Prepare Stock Pytorch - run: | - pwd - which conda && conda clean -ay - conda remove --all -y -n xpu_op_${ZE_AFFINITY_MASK} || \ - rm -rf $(dirname ${CONDA_EXE})/../envs/xpu_op_${ZE_AFFINITY_MASK} - conda create -n xpu_op_${ZE_AFFINITY_MASK} python=${{ inputs.python }} cmake ninja -y - source activate xpu_op_${ZE_AFFINITY_MASK} - cd ../ && rm -rf pytorch - pip install requests - git clone https://github.com/pytorch/pytorch pytorch - if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then - cd pytorch && git checkout $(echo ${{ inputs.pytorch }}) - # apply PRs for stock pytorch - python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py - git status && git show -s - git submodule sync && git submodule update --init --recursive - if [[ ${{ inputs.keep_torch_xpu_ops }} == 'true' ]]; then - echo "Don't replace torch-xpu-ops!" - else - rm -rf third_party/torch-xpu-ops && cp -r ../torch-xpu-ops third_party/ - # Workaround for torch-xpu-ops ci test - sed -i "s/checkout --quiet \${TORCH_XPU_OPS_COMMIT}/log -n 1/g" caffe2/CMakeLists.txt - fi - fi - - name: Triton Installation - run: | - source activate xpu_op_${ZE_AFFINITY_MASK} - cd ../pytorch - TRITON_REPO="https://github.com/intel/intel-xpu-backend-for-triton" - if [ -z ${{ inputs.triton }} ]; then - TRITON_COMMIT_ID="$(<.ci/docker/ci_commit_pins/triton-xpu.txt)" - else - TRITON_COMMIT_ID="${{ inputs.triton }}" - fi - echo ${TRITON_REPO}@${TRITON_COMMIT_ID} - if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then - pip install --force-reinstall "git+${TRITON_REPO}@${TRITON_COMMIT_ID}#subdirectory=python" - fi - - name: Download Pytorch wheel + - name: Download PyTorch Wheel if: ${{ inputs.pytorch != 'nightly_wheel' }} uses: actions/download-artifact@v4 with: name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }} - path: ${{ github.workspace }} - - name: Install Pytorch XPU + - name: Prepare Test Env run: | - source activate xpu_op_${ZE_AFFINITY_MASK} - if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then - cd ../pytorch - export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:${CONDA_PREFIX:-"$(dirname $(which conda))/../"} - pip install -r requirements.txt - pip install --force-reinstall ${{ github.workspace }}/torch*.whl - git clone https://github.com/pytorch/vision && cd vision && python setup.py install && cd .. + set -xe -o pipefail + PYTORCH_REPO="https://github.com/pytorch/pytorch.git" + if [ "${{ inputs.pytorch }}" == "nightly_wheel" ]; then + PYTORCH_VERSION="nightly" else - pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu - TORCH_COMMIT_ID=$(python -c 'import torch; print(torch.version.git_version)') - cd ../pytorch - git reset --hard && git checkout ${TORCH_COMMIT_ID} - TORCH_XPU_OPS_COMMIT=$(${{ github.workspace }}/ut_log/op_regression/op_regression_test_error.log | \ tee ${{ github.workspace }}/ut_log/op_regression/op_regression_test.log - - name: Run XPU OP Regressions test on device 1 - if: contains(inputs.ut, 'op_regression_dev1') - run: | - cd ${{ github.workspace }} - mkdir -p ut_log/op_regression_dev1 - xpu-smi discovery - source activate xpu_op_${ZE_AFFINITY_MASK} - unset ZE_AFFINITY_MASK - cd ${{ github.workspace }} - cd test/regressions - timeout 8000 pytest --timeout 600 -v test_operation_on_device_1.py \ - --junit-xml=${{ github.workspace }}/ut_log/op_regression_dev1.xml \ - 2>${{ github.workspace }}/ut_log/op_regression_dev1/op_regression_dev1_test_error.log | \ - tee ${{ github.workspace }}/ut_log/op_regression_dev1/op_regression_dev1_test.log - name: Run XPU Transformers UT if: contains(inputs.ut, 'op_transformers') run: | @@ -198,7 +125,7 @@ jobs: cd ${{ github.workspace }} mkdir -p ut_log/op_ut cd ../pytorch/third_party/torch-xpu-ops/test/xpu - timeout 10000 python run_test_with_skip.py \ + timeout 18000 python run_test_with_skip.py \ 2>${{ github.workspace }}/ut_log/op_ut/op_ut_with_skip_test_error.log | \ tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_skip_test.log cp *.xml ${{ github.workspace }}/ut_log @@ -314,15 +241,66 @@ jobs: with: name: XPU-UT-Failure-List-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_NAME }} path: ${{ github.workspace }}/ut_log/ut_failure_list.csv - + - name: Clean diskspace + if: ${{ always() }} + uses: ./.github/actions/diskspace-clean + ut_test_dev1: + runs-on: pvc_e2e + if: ${{ inputs.ut != 'xpu_distributed' && !contains(inputs.disabled_tests, 'disable_ut') }} + timeout-minutes: 600 + env: + GH_TOKEN: ${{ github.token }} + NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }} + DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }} + ut_skip_issue: 1624 + steps: + - name: Checkout Torch-xpu-ops + uses: actions/checkout@v4 + - name: Download PyTorch Wheel + if: ${{ inputs.pytorch != 'nightly_wheel' }} + uses: actions/download-artifact@v4 + with: + name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }} + - name: Prepare Test Env + run: | + set -xe -o pipefail + PYTORCH_REPO="https://github.com/pytorch/pytorch.git" + if [ "${{ inputs.pytorch }}" == "nightly_wheel" ]; then + PYTORCH_VERSION="nightly" + else + PYTORCH_VERSION="main" + fi + cd ../ + ./torch-xpu-ops/.github/scripts/setup_test_env.sh \ + --WORKSPACE="${{ github.workspace }}" \ + --CONDA_ENV="xpu_op_${ZE_AFFINITY_MASK}" \ + --PYTHON_VERSION="${{ inputs.python }}" \ + --PYTORCH_REPO="${PYTORCH_REPO}" \ + --PYTORCH_VERSION="${PYTORCH_VERSION}" \ + --TRITON_VERSION="${{ inputs.triton }}" + - name: Run XPU OP Regressions test on device 1 + if: contains(inputs.ut, 'op_regression_dev1') + run: | + set -xe -o pipefail + xpu-smi discovery + source activate xpu_op_${ZE_AFFINITY_MASK} + unset ZE_AFFINITY_MASK + cd test/regressions + timeout 8000 pytest --timeout 600 -v test_operation_on_device_1.py + - name: Clean diskspace + if: ${{ always() }} + uses: ./.github/actions/diskspace-clean ut_test_results_check: - needs: ut_test + needs: [ut_test, ut_test_dev1] runs-on: ubuntu-22.04 timeout-minutes: 30 env: GH_TOKEN: ${{ github.token }} ut_skip_issue: 1624 steps: + - name: Cleanup Workspace + run: | + rm -rf ${{ github.workspace }}/* || sudo rm -rf ${{ github.workspace }}/* - name: Set the UT name run: | echo "UT_NAME=$(echo ${{ inputs.ut }} |sed 's/,/-/g')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" @@ -350,7 +328,7 @@ jobs: echo "UT_NAME=$(echo ${{ inputs.ut }} |sed 's/,/-/g')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" for ut_suite in $(echo ${{ inputs.ut }} |sed 's/,/ /g') do - contains "op_regression,op_regression_dev1,op_transformers,op_extended,op_ut,torch_xpu" $ut_suite + contains "op_regression,op_transformers,op_extended,op_ut,torch_xpu" $ut_suite $contains_status cd ${{ github.workspace }}/ut_log/${ut_suite} gh --repo $repo issue view $ut_skip_issue --json body -q .body | sed '/^$/d' > Known_issue.log @@ -371,6 +349,9 @@ jobs: with: name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_NAME }}-checked path: ${{ github.workspace }}/ut_log + - name: Clean diskspace + if: ${{ always() }} + uses: ./.github/actions/diskspace-clean distributed_ut_test: runs-on: pytorch-06 @@ -382,87 +363,33 @@ jobs: DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }} ut_skip_issue: 1624 steps: - - name: Checkout torch-xpu-ops - uses: actions/checkout@v4 - - name: Prepare Stock Pytorch + - name: Cleanup Workspace run: | - pwd - which conda && conda clean -ay - conda remove --all -y -n xpu_op_${ZE_AFFINITY_MASK} || \ - rm -rf $(dirname ${CONDA_EXE})/../envs/xpu_op_${ZE_AFFINITY_MASK} - conda create -n xpu_op_${ZE_AFFINITY_MASK} python=${{ inputs.python }} cmake ninja -y - source activate xpu_op_${ZE_AFFINITY_MASK} - cd ../ && rm -rf pytorch - pip install requests - git clone https://github.com/pytorch/pytorch pytorch - if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then - cd pytorch && git checkout $(echo ${{ inputs.pytorch }}) - # apply PRs for stock pytorch - python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py - git status && git show -s - git submodule sync && git submodule update --init --recursive - if [[ ${{ inputs.keep_torch_xpu_ops }} == 'true' ]]; then - echo "Don't replace torch-xpu-ops!" - else - rm -rf third_party/torch-xpu-ops && cp -r ../torch-xpu-ops third_party/ - # Workaround for torch-xpu-ops ci test - sed -i "s/checkout --quiet \${TORCH_XPU_OPS_COMMIT}/log -n 1/g" caffe2/CMakeLists.txt - fi - fi - - name: Triton Installation - run: | - source activate xpu_op_${ZE_AFFINITY_MASK} - cd ../pytorch - TRITON_REPO="https://github.com/intel/intel-xpu-backend-for-triton" - if [ -z ${{ inputs.triton }} ]; then - TRITON_COMMIT_ID="$(<.ci/docker/ci_commit_pins/triton-xpu.txt)" - else - TRITON_COMMIT_ID="${{ inputs.triton }}" - fi - echo ${TRITON_REPO}@${TRITON_COMMIT_ID} - if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then - pip install --force-reinstall "git+${TRITON_REPO}@${TRITON_COMMIT_ID}#subdirectory=python" - fi - - name: Download Pytorch wheel + rm -rf ${{ github.workspace }}/* || sudo rm -rf ${{ github.workspace }}/* + - name: Checkout Torch-xpu-ops + uses: actions/checkout@v4 + - name: Download PyTorch Wheel if: ${{ inputs.pytorch != 'nightly_wheel' }} uses: actions/download-artifact@v4 with: name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }} - path: ${{ github.workspace }} - - name: Install Pytorch XPU + - name: Prepare Test Env run: | - source activate xpu_op_${ZE_AFFINITY_MASK} - if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then - cd ../pytorch - export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:${CONDA_PREFIX:-"$(dirname $(which conda))/../"} - pip install -r requirements.txt - pip install --force-reinstall ${{ github.workspace }}/torch*.whl - git clone https://github.com/pytorch/vision && cd vision && python setup.py install && cd .. + set -xe -o pipefail + PYTORCH_REPO="https://github.com/pytorch/pytorch.git" + if [ "${{ inputs.pytorch }}" == "nightly_wheel" ]; then + PYTORCH_VERSION="nightly" else - pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu - TORCH_COMMIT_ID=$(python -c 'import torch; print(torch.version.git_version)') - cd ../pytorch - git reset --hard && git checkout ${TORCH_COMMIT_ID} - TORCH_XPU_OPS_COMMIT=$(> "${GITHUB_ENV}" @@ -549,3 +480,6 @@ jobs: with: name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-xpu_distributed-checked path: ${{ github.workspace }}/ut_log + - name: Clean diskspace + if: ${{ always() }} + uses: ./.github/actions/diskspace-clean diff --git a/.github/workflows/_performance_comparison.yml b/.github/workflows/_performance_comparison.yml index 9891da8d2..19e179b7a 100644 --- a/.github/workflows/_performance_comparison.yml +++ b/.github/workflows/_performance_comparison.yml @@ -20,9 +20,6 @@ jobs: GH_TOKEN: ${{ github.token }} runs-on: ubuntu-latest steps: - - name: Cleanup workspace - run: | - rm -rf ./target ./baseline - name: Setup python uses: actions/setup-python@v5 with: @@ -41,3 +38,6 @@ jobs: run: | python -m pip install pandas python .github/scripts/perf_comparison.py --xpu target --refer baseline >> ${GITHUB_STEP_SUMMARY} + - name: Clean diskspace + if: ${{ always() }} + uses: ./.github/actions/diskspace-clean diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml index ed50f9371..b2bb88efa 100644 --- a/.github/workflows/nightly_ondemand.yml +++ b/.github/workflows/nightly_ondemand.yml @@ -74,7 +74,6 @@ jobs: keep_torch_xpu_ops: ${{ github.event_name == 'schedule' && 'false' || inputs.keep_torch_xpu_ops }} python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }} runner: pvc_e2e - update_lkg: 'true' Linux-Nightly-Ondemand-UT-Tests: if: ${{ github.event_name == 'schedule' || inputs.ut != '' }} @@ -86,7 +85,7 @@ jobs: ut: ${{ github.event_name == 'schedule' && 'op_regression,op_regression_dev1,op_transformers,op_extended,op_ut' || inputs.ut }} pytorch: ${{ needs.Linux-Nightly-Ondemand-Build.outputs.torch_commit_id }} python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }} - triton: ${{ github.event_name == 'schedule' && '' || inputs.triton }} + triton: ${{ (inputs.triton != '' && inputs.triton) || 'pinned' }} runner: linux.idc.xpu Linux-Nightly-Ondemand-E2E-Tests: @@ -117,33 +116,30 @@ jobs: TRITON_COMMIT_ID: ${{ steps.pinned.outputs.TRITON_COMMIT_ID }} TIMEOUT_MODELS: ${{ steps.summary.outputs.TIMEOUT_MODELS }} steps: - - name: Checkout torch-xpu-ops + - name: Checkout Torch-xpu-ops uses: actions/checkout@v4 - - name: Prepare Conda ENV - run: | - which conda && conda clean -ay - conda remove --all -y -n e2e_ci || rm -rf $(dirname ${CONDA_EXE})/../envs/e2e_ci - conda create -n e2e_ci python=${{ env.python }} cmake ninja -y - source activate e2e_ci - pip install pandas scipy psutil - - name: Prepare Stock Pytorch + - name: Download Pytorch wheel + uses: actions/download-artifact@v4 + with: + name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }} + - name: Prepare Test Env run: | - pwd - cd ../ && rm -rf pytorch - source activate e2e_ci - git clone https://github.com/pytorch/pytorch pytorch - cd pytorch && git checkout $(echo ${{ env.pytorch }} |awk '{print $1}') - # apply PRs for stock pytorch - pip install requests - python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py - git status && git show -s - if [[ ${{ env.keep_torch_xpu_ops }} == 'true' ]]; then - echo "Don't replace torch-xpu-ops!" + set -xe -o pipefail + PYTORCH_REPO="https://github.com/pytorch/pytorch.git" + PYTORCH_VERSION="main" + cd ../ + if [ -z ${{ inputs.triton }} ];then + TRITON_VERSION="pinned" else - rm -rf third_party/torch-xpu-ops && cp -r ../torch-xpu-ops third_party/ - # Workaround for torch-xpu-ops ci test - sed -i "s/checkout --quiet \${TORCH_XPU_OPS_COMMIT}/log -n 1/g" caffe2/CMakeLists.txt + TRITON_VERSION="${{ inputs.triton }}" fi + ./torch-xpu-ops/.github/scripts/setup_test_env.sh \ + --WORKSPACE="${{ github.workspace }}" \ + --CONDA_ENV="e2e_ci" \ + --PYTHON_VERSION="${{ inputs.python }}" \ + --PYTORCH_REPO="${PYTORCH_REPO}" \ + --PYTORCH_VERSION="${PYTORCH_VERSION}" \ + --TRITON_VERSION="${TRITON_VERSION}" - name: Identify pinned versions id: pinned run: | @@ -169,31 +165,10 @@ jobs: echo "OS_PRETTY_NAME=${PRETTY_NAME}" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" echo "GCC_VERSION=$(gcc -dumpversion)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" echo ${GITHUB_ENV} - - name: Triton Installation - run: | - source activate e2e_ci - cd ../pytorch - TRITON_REPO="https://github.com/intel/intel-xpu-backend-for-triton" - echo ${TRITON_REPO}@${TRITON_COMMIT_ID} - pip install --force-reinstall "git+${TRITON_REPO}@${TRITON_COMMIT_ID}#subdirectory=python" - - name: Download Pytorch wheel - if: ${{ inputs.pytorch != 'nightly_wheel' }} - uses: actions/download-artifact@v4 - with: - name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }} - path: ${{ github.workspace }} - - name: Install Pytorch XPU - run: | - source activate e2e_ci - cd ../pytorch - pip install -r requirements.txt - pip install --force-reinstall ${{ github.workspace }}/torch*.whl - name: Show GITHUB_ENV run: | echo "$GITHUB_ENV" rm -rf ../pytorch/inductor_log - rm -rf /tmp/torchinductor_* - rm -rf ~/.triton/cache # Nihglty launch - name: Nightly Huggingface FP32/BF16/FP16 Inference & Training Accuracy Test @@ -352,6 +327,9 @@ jobs: gh --repo ${GITHUB_REPOSITORY} issue view ${reference_issue} --json body -q .body | \ sed "s/Inductor-${{ env.run_type }}-Rolling-XPU-E2E:.*/Inductor-${{ env.run_type }}-Rolling-XPU-E2E: ${GITHUB_RUN_ID}/" | sed '/^$/d' > new_body.txt gh --repo ${GITHUB_REPOSITORY} issue edit ${reference_issue} --body-file new_body.txt + - name: Clean diskspace + if: ${{ always() }} + uses: ./.github/actions/diskspace-clean Windows-Nightly-Ondemand-UT-Tests: if: ${{ github.event_name == 'schedule' || inputs.ut != '' }} diff --git a/.github/workflows/nightly_ondemand_rolling.yml b/.github/workflows/nightly_ondemand_rolling.yml index 9cc0f73ec..150cfc5da 100644 --- a/.github/workflows/nightly_ondemand_rolling.yml +++ b/.github/workflows/nightly_ondemand_rolling.yml @@ -75,7 +75,6 @@ jobs: python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }} driver: rolling runner: pvc_rolling - update_lkg: 'true' Linux-Nightly-Ondemand-UT-Tests-Rolling: if: ${{ github.event_name == 'schedule' || inputs.ut != '' }} @@ -87,7 +86,7 @@ jobs: ut: ${{ github.event_name == 'schedule' && 'op_regression,op_regression_dev1,op_transformers,op_extended,op_ut' || inputs.ut }} pytorch: ${{ needs.Linux-Nightly-Ondemand-Build-Rolling.outputs.torch_commit_id }} python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }} - triton: ${{ github.event_name == 'schedule' && '' || inputs.triton }} + triton: ${{ (inputs.triton == '' && 'pinned') || inputs.triton }} driver: rolling runner: pvc_rolling @@ -99,7 +98,7 @@ jobs: keep_torch_xpu_ops: ${{ github.event_name == 'schedule' && 'false' || inputs.keep_torch_xpu_ops }} pytorch: ${{ needs.Linux-Nightly-Ondemand-Build-Rolling.outputs.torch_commit_id }} python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }} - triton: ${{ github.event_name == 'schedule' && '' || inputs.triton }} + triton: ${{ (inputs.triton != '' && inputs.triton) || 'pinned' }} driver: rolling runner: pvc_rolling @@ -133,33 +132,30 @@ jobs: TRITON_COMMIT_ID: ${{ steps.pinned.outputs.TRITON_COMMIT_ID }} TIMEOUT_MODELS: ${{ steps.summary.outputs.TIMEOUT_MODELS }} steps: - - name: Checkout torch-xpu-ops + - name: Checkout Torch-xpu-ops uses: actions/checkout@v4 - - name: Prepare Conda ENV - run: | - which conda && conda clean -ay - conda remove --all -y -n e2e_ci || rm -rf $(dirname ${CONDA_EXE})/../envs/e2e_ci - conda create -n e2e_ci python=${{ env.python }} cmake ninja -y - source activate e2e_ci - pip install pandas scipy psutil - - name: Prepare Stock Pytorch + - name: Download Pytorch wheel + uses: actions/download-artifact@v4 + with: + name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }} + - name: Prepare Test Env run: | - pwd - cd ../ && rm -rf pytorch - source activate e2e_ci - git clone https://github.com/pytorch/pytorch pytorch - cd pytorch && git checkout $(echo ${{ env.pytorch }} |awk '{print $1}') - # apply PRs for stock pytorch - pip install requests - python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py - git status && git show -s - if [[ ${{ env.keep_torch_xpu_ops }} == 'true' ]]; then - echo "Don't replace torch-xpu-ops!" + set -xe -o pipefail + PYTORCH_REPO="https://github.com/pytorch/pytorch.git" + PYTORCH_VERSION="main" + cd ../ + if [ -z ${{ inputs.triton }} ];then + TRITON_VERSION="pinned" else - rm -rf third_party/torch-xpu-ops && cp -r ../torch-xpu-ops third_party/ - # Workaround for torch-xpu-ops ci test - sed -i "s/checkout --quiet \${TORCH_XPU_OPS_COMMIT}/log -n 1/g" caffe2/CMakeLists.txt + TRITON_VERSION="${{ inputs.triton }}" fi + ./torch-xpu-ops/.github/scripts/setup_test_env.sh \ + --WORKSPACE="${{ github.workspace }}" \ + --CONDA_ENV="e2e_ci" \ + --PYTHON_VERSION="${{ inputs.python }}" \ + --PYTORCH_REPO="${PYTORCH_REPO}" \ + --PYTORCH_VERSION="${PYTORCH_VERSION}" \ + --TRITON_VERSION="${TRITON_VERSION}" - name: Identify pinned versions id: pinned run: | @@ -185,31 +181,10 @@ jobs: echo "OS_PRETTY_NAME=${PRETTY_NAME}" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" echo "GCC_VERSION=$(gcc -dumpversion)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" echo ${GITHUB_ENV} - - name: Triton Installation - run: | - source activate e2e_ci - cd ../pytorch - TRITON_REPO="https://github.com/intel/intel-xpu-backend-for-triton" - echo ${TRITON_REPO}@${TRITON_COMMIT_ID} - pip install --force-reinstall "git+${TRITON_REPO}@${TRITON_COMMIT_ID}#subdirectory=python" - - name: Download Pytorch wheel - if: ${{ inputs.pytorch != 'nightly_wheel' }} - uses: actions/download-artifact@v4 - with: - name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }} - path: ${{ github.workspace }} - - name: Install Pytorch XPU - run: | - source activate e2e_ci - cd ../pytorch - pip install -r requirements.txt - pip install --force-reinstall ${{ github.workspace }}/torch*.whl - name: Show GITHUB_ENV run: | echo "$GITHUB_ENV" rm -rf ../pytorch/inductor_log - rm -rf /tmp/torchinductor_* - rm -rf ~/.triton/cache # Nihglty launch - name: Nightly Huggingface FP32/BF16/FP16 Inference & Training Accuracy Test @@ -377,6 +352,9 @@ jobs: gh --repo ${GITHUB_REPOSITORY} issue view ${reference_issue} --json body -q .body | \ sed "s/Inductor-${{ env.run_type }}-Rolling-XPU-E2E:.*/Inductor-${{ env.run_type }}-Rolling-XPU-E2E: ${GITHUB_RUN_ID}/" | sed '/^$/d' > new_body.txt gh --repo ${GITHUB_REPOSITORY} issue edit ${reference_issue} --body-file new_body.txt + - name: Clean diskspace + if: ${{ always() }} + uses: ./.github/actions/diskspace-clean Tests-Failure-And-Report: if: ${{ ! cancelled() }} diff --git a/.github/workflows/nightly_ondemand_whl.yml b/.github/workflows/nightly_ondemand_whl.yml index 3dacbd66c..9c9272d34 100644 --- a/.github/workflows/nightly_ondemand_whl.yml +++ b/.github/workflows/nightly_ondemand_whl.yml @@ -13,6 +13,11 @@ on: type: string default: 'nightly' description: Pytorch branch/commit + triton: + required: false + type: string + default: '' + description: Triton commit. Use pytorch pined commit by default ut: required: false type: string @@ -58,6 +63,7 @@ jobs: if: ${{ (github.event_name == 'schedule' || inputs.ut != '') && github.repository_owner == 'intel' }} uses: ./.github/workflows/_linux_ut.yml with: + triton: ${{ github.event_name == 'schedule' && '' || inputs.triton }} ut: ${{ github.event_name == 'schedule' && 'op_regression,op_regression_dev1,op_transformers,op_extended,op_ut' || inputs.ut }} python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }} pytorch: nightly_wheel @@ -90,39 +96,21 @@ jobs: GCC_VERSION: ${{ steps.pinned.outputs.GCC_VERSION }} TIMEOUT_MODELS: ${{ steps.summary.outputs.TIMEOUT_MODELS }} steps: - - name: Checkout torch-xpu-ops + - name: Checkout Torch-xpu-ops uses: actions/checkout@v4 - - name: Prepare Conda ENV + - name: Prepare Test Env run: | - which conda && conda clean -ay - conda remove --all -y -n e2e_ci || rm -rf $(dirname ${CONDA_EXE})/../envs/e2e_ci - conda create -n e2e_ci python=${{ env.python }} cmake ninja -y - source activate e2e_ci - pip install pandas scipy psutil - - name: Prepare Stock Pytorch - id: installed - run: | - pwd - source activate e2e_ci - pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu - echo "TORCH_BRANCH_ID=$(python -c 'import torch; print(torch.__version__)')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" - TORCH_COMMIT_ID=$(python -c 'import torch; print(torch.version.git_version)') - echo "TORCH_COMMIT_ID=${TORCH_COMMIT_ID}" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" - cd ../ && rm -rf pytorch - git clone https://github.com/pytorch/pytorch pytorch - cd pytorch && git checkout ${TORCH_COMMIT_ID} - # apply PRs for stock pytorch - pip install requests - git status && git show -s - pip install -r requirements.txt - TORCH_XPU_OPS_COMMIT=$(> "${GITHUB_ENV}" - rm -rf third_party/torch-xpu-ops - git clone https://github.com/intel/torch-xpu-ops.git third_party/torch-xpu-ops - cd third_party/torch-xpu-ops - git checkout ${TORCH_XPU_OPS_COMMIT} - cd ../../ - python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py + set -xe -o pipefail + PYTORCH_REPO="https://github.com/pytorch/pytorch.git" + PYTORCH_VERSION="nightly" + cd ../ + ./torch-xpu-ops/.github/scripts/setup_test_env.sh \ + --WORKSPACE="${{ github.workspace }}" \ + --CONDA_ENV="e2e_ci" \ + --PYTHON_VERSION="${{ inputs.python }}" \ + --PYTORCH_REPO="${PYTORCH_REPO}" \ + --PYTORCH_VERSION="${PYTORCH_VERSION}" \ + --TRITON_VERSION="${{ inputs.triton }}" - name: Identify pinned versions id: pinned run: | @@ -147,8 +135,6 @@ jobs: run: | echo "$GITHUB_ENV" rm -rf ../pytorch/inductor_log - rm -rf /tmp/torchinductor_* - rm -rf ~/.triton/cache # Nihglty launch - name: Nightly Huggingface FP32/BF16/FP16 Inference & Training Accuracy Test @@ -316,6 +302,9 @@ jobs: gh --repo ${GITHUB_REPOSITORY} issue view ${reference_issue} --json body -q .body | \ sed "s/Inductor-${{ env.run_type }}-Rolling-XPU-E2E:.*/Inductor-${{ env.run_type }}-Rolling-XPU-E2E: ${GITHUB_RUN_ID}/" | sed '/^$/d' > new_body.txt gh --repo ${GITHUB_REPOSITORY} issue edit ${reference_issue} --body-file new_body.txt + - name: Clean diskspace + if: ${{ always() }} + uses: ./.github/actions/diskspace-clean Tests-Failure-And-Report: if: ${{ ! cancelled() }} diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index ecbb8dbc2..2d7a17099 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -106,6 +106,7 @@ jobs: needs: [preci-conditions-filter, preci-linux-build] uses: ./.github/workflows/_linux_ut.yml with: + triton: pinned disabled_tests: ${{ needs.preci-conditions-filter.outputs.disabled_tests }} ut: op_regression,op_regression_dev1,op_transformers,op_extended,op_ut,xpu_distributed runner: linux.idc.xpu @@ -120,67 +121,38 @@ jobs: reference_issue: 1645 timeout-minutes: 300 steps: - - name: Checkout torch-xpu-ops + - name: Checkout Torch-xpu-ops uses: actions/checkout@v4 - - name: Prepare Conda ENV - run: | - which conda && conda clean -ay - conda remove --all -y -n e2e_ci || rm -rf $(dirname ${CONDA_EXE})/../envs/e2e_ci - conda create -n e2e_ci python=3.10 cmake ninja -y - source activate e2e_ci - pip install pandas scipy psutil requests - name: Download Pytorch wheel uses: actions/download-artifact@v4 with: - name: Torch-XPU-Wheel-${{ github.event.pull_request.number }} - - name: Install Pytorch XPU - run: | - source activate e2e_ci - pip install --force-reinstall ${{ github.workspace }}/torch*.whl - TORCH_COMMIT_ID=$(python -c 'import torch; print(torch.version.git_version)') - cd ../ && rm -rf pytorch - git clone https://github.com/pytorch/pytorch pytorch - cd pytorch && git checkout ${TORCH_COMMIT_ID} - pip install -r requirements.txt - # apply PRs for stock pytorch - # https://github.com/pytorch/pytorch/pull/152940 internal use only for subset model list - python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py -e https://github.com/pytorch/pytorch/pull/152940 - git show -s && git status && git diff - - name: Triton Installation + name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }} + - name: Prepare Test Env run: | - source activate e2e_ci - cd ../pytorch - TRITON_REPO="https://github.com/intel/intel-xpu-backend-for-triton" - TRITON_PINNED_COMMIT=$(cat .ci/docker/ci_commit_pins/triton-xpu.txt) - echo ${TRITON_REPO}@${TRITON_PINNED_COMMIT} - pip install --force-reinstall "git+${TRITON_REPO}@${TRITON_PINNED_COMMIT}#subdirectory=python" + set -xe -o pipefail + PYTORCH_REPO="https://github.com/pytorch/pytorch.git" + PYTORCH_VERSION="main" + cd ../ + ./torch-xpu-ops/.github/scripts/setup_test_env.sh \ + --WORKSPACE="${{ github.workspace }}" \ + --CONDA_ENV="e2e_ci" \ + --PYTHON_VERSION="3.10" \ + --PYTORCH_REPO="${PYTORCH_REPO}" \ + --PYTORCH_VERSION="${PYTORCH_VERSION}" \ + --TRITON_VERSION="pinned" - name: Identify pinned versions run: | cd ../pytorch - echo "TORCH_BRANCH_ID=$(git rev-parse --abbrev-ref HEAD)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" - echo "TORCH_COMMIT_ID=$(git rev-parse HEAD)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" - echo "TRITON_COMMIT_ID=$(<.ci/docker/ci_commit_pins/triton-xpu.txt)" >> "${GITHUB_ENV}" echo "TORCHVISION_COMMIT_ID=$(<.github/ci_commit_pins/vision.txt)" >> "${GITHUB_ENV}" echo "TORCHBENCH_COMMIT_ID=$(<.github/ci_commit_pins/torchbench.txt)" >> "${GITHUB_ENV}" echo "TORCHAUDIO_COMMIT_ID=$(<.github/ci_commit_pins/audio.txt)" >> "${GITHUB_ENV}" echo "TRANSFORMERS_VERSION=$(<.ci/docker/ci_commit_pins/huggingface.txt)" >> "${GITHUB_ENV}" echo "TIMM_COMMIT_ID=$(<.ci/docker/ci_commit_pins/timm.txt)" >> "${GITHUB_ENV}" - . /etc/os-release - echo "OS_PRETTY_NAME=${PRETTY_NAME}" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" - echo "GCC_VERSION=$(gcc -dumpversion)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" - source ../torch-xpu-ops/.github/scripts/env.sh - echo "DRIVER_VERSION=$(sycl-ls |grep 'opencl:gpu' |awk '{print $NF}' |sort |uniq -c |sed 's/ //g;s/\[/*[/')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" - echo "KERNEL_VERSION=$(uname -rv 2>&1)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" - echo "BUNDLE_VERSION=$(icpx --version 2>&1 |grep 'DPC++/C++' |sed 's/.*(//;s/).*//')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" - name: Torch Config run: | echo "$GITHUB_ENV" rm -rf ../pytorch/inductor_log - rm -rf /tmp/torchinductor_* - cd .. - source activate e2e_ci - python -c "import triton; print(triton.__version__)" - python pytorch/torch/utils/collect_env.py + - name: Huggingface BF16 Training Accuracy Test uses: ./.github/actions/inductor-xpu-e2e-test with: @@ -251,6 +223,9 @@ jobs: with: name: Inductor-CI-XPU-E2E-Data-${{ github.event.pull_request.number || github.sha }} path: ${{ github.workspace }}/upload_files + - name: Clean diskspace + if: ${{ always() }} + uses: ./.github/actions/diskspace-clean preci-windows: name: preci-windows