debug

dvrogozh · dvrogozh · commit 3dd39dcd15f0 · 2025-04-18T19:34:47.000-07:00
Signed-off-by: Dmitry Rogozhkin &lt;dmitry.v.rogozhkin@intel.com&gt;
diff --git a/.github/actions/print-environment/action.yml b/.github/actions/print-environment/action.yml
@@ -16,6 +16,11 @@ inputs:
     type: string
     default: ''
     description: "Space separated list of PyPi packages to evaluate"
+  to:
+    required: false
+    type: string
+    default: ''
+    description: "File to print environment to"
 
 runs:
   using: composite
@@ -26,6 +31,11 @@ runs:
         if [ -n "${{ inputs.conda }}" ]; then
           source activate ${{ inputs.conda }}
         fi
+        to=$GITHUB_STEP_SUMMARY
+        if [ -n "${{ inputs.to }}" ]; then
+          to="${{ inputs.to }}"
+          mkdir -p $(dirname $to)
+        fi
         {
           echo "### Environment"
           echo "| | |"
@@ -71,4 +81,4 @@ runs:
           echo "| jobs.$GITHUB_JOB.env.NEOReadDebugKeys | $NEOReadDebugKeys |"
           echo "| jobs.$GITHUB_JOB.env.PYTORCH_ENABLE_XPU_FALLBACK | $PYTORCH_ENABLE_XPU_FALLBACK |"
           echo "| jobs.$GITHUB_JOB.env.PYTORCH_DEBUG_XPU_FALLBACK | $PYTORCH_DEBUG_XPU_FALLBACK |"
-        } >> $GITHUB_STEP_SUMMARY
+        } >> $to
diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml
@@ -13,11 +13,6 @@ on:
       - '.github/workflows/_linux_transformers.yml'
   workflow_dispatch:
     inputs:
-      pytorch:
-        required: false
-        type: string
-        default: 'nightly'
-        description: Pytorch branch/commit
       python:
         required: false
         type: string
@@ -51,7 +46,6 @@ env:
   NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
   DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
   python: ${{ inputs.python != '' && inputs.python || '3.10' }}
-  pytorch: ${{ inputs.pytorch != '' && inputs.pytorch || 'nightly' }}
   transformers: ${{ inputs.transformers != '' && inputs.transformers || 'v4.49.0' }}
   PACKAGES: |
     espeak-ng
@@ -65,64 +59,27 @@ env:
     libswresample-dev
     libswscale-dev
   PYTEST_TIMEOUT: 600
+  TORCH_INDEX: '--pre --index-url https://download.pytorch.org/whl/nightly/xpu'
 
 jobs:
   prepare:
     runs-on: ${{ inputs.runner != '' && inputs.runner || 'linux.idc.xpu' }}
+    outputs:
+      torch: ${{ steps.getver.outputs.torch }}
+      torchvision: ${{ steps.getver.outputs.torchvision }}
+      torchaudio: ${{ steps.getver.outputs.torchaudio }}
     steps:
-      - name: Checkout torch-xpu-ops
-        uses: actions/checkout@v4
-        with:
-          path: torch-xpu-ops
-      - name: Checkout Transformers
-        uses: actions/checkout@v4
-        with:
-          repository: huggingface/transformers
-          ref: ${{ env.transformers }}
-          path: transformers
-      - name: Prepare OS environment
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y $PACKAGES
-          git lfs install
-      - name: Create unique Conda ENV name
+      - id: getver
         run: |
-          echo "CONDA_ENV_NAME=hf_transformers_test_${ZE_AFFINITY_MASK}" >> $GITHUB_ENV
-      - name: Prepare Conda ENV
-        run: |
-          echo "Using Conda ENV name: $CONDA_ENV_NAME"
-          which conda && conda clean -ay
-          conda remove --all -y -n $CONDA_ENV_NAME || rm -rf $(dirname ${CONDA_EXE})/../envs/$CONDA_ENV_NAME
-          conda create -y -n $CONDA_ENV_NAME python=${{ env.python }}
-          source activate $CONDA_ENV_NAME
-          pip install junitparser pytest-timeout
-      - name: Prepare Stock XPU Pytorch
-        run: |
-          pwd
-          source activate $CONDA_ENV_NAME
-          if [ -z "${{ inputs.nightly_whl }}" ]; then
-            pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu
-          else
-            pip install torch==$(echo ${{ inputs.nightly_whl }}) torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu
-          fi
-      - name: Prepare Transformers
-        run: |
-          source activate $CONDA_ENV_NAME
-          cd transformers
-          pip install -e .
-          pip install -e ".[dev-torch,testing,video]"
-      - name: Export environment
-          pip freeze --exclude-editable | tee ${{ github.workspace }}/transformers/tests_log/pip_freeze.txt
-      - name: Upload pip_freeze
-        if: ${{ ! cancelled() }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: pip_freeze
-          path: |
-            ${{ github.workspace }}/transformers/tests_log/pip_freeze.txt
+          torch=$(pip index versions torch $TORCH_INDEX | grep torch | sed -E 's/.*\((.*)\)/\1/')
+          torchvision=$(pip index versions torchvision $TORCH_INDEX | grep torch | sed -E 's/.*\((.*)\)/\1/')
+          torchaudio=$(pip index versions torchaudio $TORCH_INDEX | grep torch | sed -E 's/.*\((.*)\)/\1/')
+          echo "torch=$torch" | tee "$GITHUB_OUTPUT"
+          echo "torchvision=$torchvision" | tee "$GITHUB_OUTPUT"
+          echo "torchaudio=$torchaudio" | tee "$GITHUB_OUTPUT"
 
   tests:
-    needs: [prepare]
+    needs: prepare
     runs-on: ${{ inputs.runner != '' && inputs.runner || 'linux.idc.xpu' }}
     strategy:
       matrix:
@@ -131,8 +88,8 @@ jobs:
           # * https://github.com/huggingface/transformers/issues/36267 (marian tests)
           ##- cmd: '--ignore=tests/models/marian/test_modeling_marian.py -k backbone tests'
           ##  test_case: 'tests_backbone'
-          - cmd: "tests/*.py"
-            test_case: "tests_py"
+          - test_case: "tests_py"
+            cmd: "tests/*.py"
           # Excluding tests due to:
           # * torch.distributed.* not yet supported by XPU
           ##- cmd: 'tests/generation'
@@ -156,16 +113,13 @@ jobs:
           # * Network proxy connection issue, reason unknown
           # *'tests/utils/test_import_utils.py' invalidates state of the test engine causing
           #   next tests to fail. See: https://github.com/huggingface/transformers/issues/36267
-          - cmd: '--ignore=tests/utils/test_import_utils.py tests/utils'
+          - test_case: 'tests_utils'
+            cmd: '--ignore=tests/utils/test_import_utils.py tests/utils'
             filter: 'not test_load_img_url_timeout'
-            test_case: 'tests_utils'
     env:
       PYTORCH_DEBUG_XPU_FALLBACK: '1'
       TRANSFORMERS_TEST_DEVICE_SPEC: 'spec.py'
     steps:
-      - uses: actions/download-artifact@v4
-        with:
-          name: pip_freeze
       - name: Checkout torch-xpu-ops
         uses: actions/checkout@v4
         with:
@@ -191,28 +145,34 @@ jobs:
           conda remove --all -y -n $CONDA_ENV_NAME || rm -rf $(dirname ${CONDA_EXE})/../envs/$CONDA_ENV_NAME
           conda create -y -n $CONDA_ENV_NAME python=${{ env.python }}
           source activate $CONDA_ENV_NAME
-          find . -name "pip_freeze*"
-          pip install -r pip_freeze.txt
+          pip install junitparser pytest-timeout
+      - name: Prepare Stock XPU Pytorch
+        run: |
+          source activate $CONDA_ENV_NAME
+          pip install $TORCH_INDEX \
+            torch==${{ needs.prepare.outputs.torch }} \
+            torchvision==${{ needs.prepare.outputs.torchvision }} \
+            torchaudio==${{ needs.prepare.outputs.torchaudio }}
       - name: Prepare Transformers
         run: |
           pwd
           source activate $CONDA_ENV_NAME
           cd transformers
           pip install -e .
           pip install -e ".[dev-torch,testing,video]"
-          rm -rf tests_log && mkdir -p tests_log
+          rm -rf logs && mkdir -p logs
           rm -rf reports
           cp ${{ github.workspace }}/torch-xpu-ops/.github/scripts/spec.py ./
       - name: Report installed versions
         run: |
           source activate $CONDA_ENV_NAME
-          LOGS_DIR="${{ github.workspace }}/transformers/test_logs/${{matrix.test.test_case}}"
+          LOGS_DIR="${{ github.workspace }}/transformers/logs/${{matrix.test.test_case}}"
           echo "pip installed packages:"
-          pip list | tee $LOGS_DIR/pip_list.txt
+          pip list | tee "$LOGS_DIR/pip_list-${{ matrix.test.test_case }}.txt"
           echo "lspci gpu devices:"
-          lspci -d ::0380 | tee $LOGS_DIR/lspci_0380.txt
+          lspci -d ::0380 | tee "$LOGS_DIR/lspci_0380-${{ matrix.test.test_case }}.txt"
           echo "GPU render nodes:"
-          cat /sys/class/drm/render*/device/device | tee $LOGS_DIR/device_IDs.txt
+          cat /sys/class/drm/render*/device/device | tee "$LOGS_DIR/device_IDs-${{ matrix.test.test_case }}.txt"
           echo "xpu-smi output:"
           xpu-smi discovery -y --json --dump -1
       - name: Sanity check installed packages
@@ -244,28 +204,51 @@ jobs:
         run: |
           du -sh ${{ env.HF_HOME }} || true
           rm -rf ${{ env.HF_HOME }}
-      - name: Upload Test log
+      - name: Print environment
+        if: ${{ ! cancelled() }}
+        uses: ./torch-xpu-ops/.github/actions/print-environment
+        with:
+          conda: $CONDA_ENV_NAME
+          pip_packages: 'accelerate transformers'
+          to: 'transformers/logs/environment-${{ matrix.test.test_case }}.md'
+      - name: Upload reports
         if: ${{ ! cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.test.test_case }}-${{ github.event.pull_request.number || github.sha }}
-          path: |
-            ${{ github.workspace }}/transformers/reports
-            ${{ github.workspace }}/transformers/tests_log
+          name: reports-${{ matrix.test.test_case }}-${{ github.event.pull_request.number || github.sha }}
+          path: ${{ github.workspace }}/transformers/reports
+      - name: Upload logs
+        if: ${{ ! cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-${{ matrix.test.test_case }}-${{ github.event.pull_request.number || github.sha }}
+          path: ${{ github.workspace }}/transformers/logs
 
   report:
-    needs: [tests]
+    needs: tests
+    if: "always()"
     runs-on: ${{ inputs.runner != '' && inputs.runner || 'linux.idc.xpu' }}
     steps:
-      - name: Download artifacts
+      - name: Download reports
+        uses: actions/download-artifact@v4
+        with:
+          name: 'reports-*'
+          path: 'transformers/reports/'
+      - name: Download logs
+        if: ${{ ! cancelled() }}
         uses: actions/download-artifact@v4
+        with:
+          name: 'logs-*'
+          path: 'transformers/logs/'
       - name: Checkout torch-xpu-ops
+        if: ${{ ! cancelled() }}
         uses: actions/checkout@v4
         with:
           path: torch-xpu-ops
       - name: Print results table
         if: ${{ ! cancelled() }}
         run: |
+          ls transformers/reports/
           find . -name "*.xml"
           # Helper function to return number preceeding given pattern, i.e:
           #   === 25 failed, 11 warnings, 0 errors ===
@@ -356,15 +339,8 @@ jobs:
           } >> $GITHUB_STEP_SUMMARY
       - name: Print environment
         if: ${{ ! cancelled() }}
-        uses: ./torch-xpu-ops/.github/actions/print-environment
-        with:
-          conda: $CONDA_ENV_NAME
-          pip_packages: 'accelerate transformers'
-      - name: Upload Test log
-        if: ${{ ! cancelled() }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: Torch-XPU-Transformers-Log-${{ github.event.pull_request.number || github.sha }}
-          path: | 
-            ${{ github.workspace }}/transformers/reports
-            ${{ github.workspace }}/transformers/tests_log
+        run: |
+          environment=$(find transformers/logs -name "environment-*.md" | head -1)
+          cat $environment >> $GITHUB_STEP_SUMMARY
+          # we expect envinments to be identical
+          diff $(find transformers/logs -name "environment-*.md")
diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml
