Skip to content

Commit 5491aa6

Browse files
RUIJIEZHONG66166RUIJIEZHONG66166
committed
[CI] Enable matirx test for UT
1 parent 02cc63f commit 5491aa6

File tree

1 file changed

+167
-151
lines changed

1 file changed

+167
-151
lines changed

‎.github/workflows/_linux_ut.yml

Lines changed: 167 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,136 @@ jobs:
5151
NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
5252
DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
5353
ut_skip_issue: 1624
54+
strategy:
55+
matrix:
56+
test:
57+
- name: 'op_regression'
58+
condition: "contains(inputs.ut, 'op_regression')"
59+
directory: 'test/regressions'
60+
command: 'pytest --timeout 600 -v --junit-xml=${{ github.workspace }}/ut_log/op_regression.xml'
61+
log_prefix: 'op_regression'
62+
timeout: 8000
63+
env: {}
64+
additional_steps: |
65+
pip install pytest pytest-timeout
66+
- name: 'op_regression_dev1'
67+
condition: "contains(inputs.ut, 'op_regression_dev1')"
68+
directory: 'test/regressions'
69+
command: 'pytest --timeout 600 -v test_operation_on_device_1.py --junit-xml=${{ github.workspace }}/ut_log/op_regression_dev1.xml'
70+
log_prefix: 'op_regression_dev1'
71+
timeout: 8000
72+
env:
73+
ZE_AFFINITY_MASK: 'unset'
74+
additional_steps: |
75+
pip install pytest pytest-timeout
76+
- name: 'op_transformers'
77+
condition: "contains(inputs.ut, 'op_transformers')"
78+
directory: '../pytorch/'
79+
command: 'pytest --timeout 600 -v test/test_transformers.py -k xpu --junit-xml=${{ github.workspace }}/ut_log/op_transformers.xml'
80+
log_prefix: 'op_transformers'
81+
timeout: 3600
82+
env:
83+
PYTORCH_TEST_WITH_SLOW: '1'
84+
additional_steps: |
85+
pip install pytest pytest-timeout
86+
- name: 'op_extended'
87+
condition: "contains(inputs.ut, 'op_extended')"
88+
directory: '../pytorch/third_party/torch-xpu-ops/test/xpu/extended/'
89+
command: 'python run_test_with_skip.py'
90+
log_prefix: 'op_extended'
91+
timeout: 10000
92+
env:
93+
PYTORCH_TEST_WITH_SLOW: '1'
94+
xml_post_processing: |
95+
cp op_extended.xml ${{ github.workspace }}/ut_log
96+
- name: 'op_ut'
97+
condition: "contains(inputs.ut, 'op_ut')"
98+
directory: '../pytorch/third_party/torch-xpu-ops/test/xpu'
99+
log_prefix: 'op_ut'
100+
commands:
101+
- command: 'python run_test_with_skip.py'
102+
log_suffix: 'with_skip'
103+
xml_post_processing: |
104+
cp *.xml ${{ github.workspace }}/ut_log
105+
find op_ut_with_skip_nn op_ut_with_skip_quantization/core -type f -exec sh -c '
106+
dir_path=$(dirname "$1");
107+
case "$dir_path" in
108+
*"op_ut_with_skip_quantization/core"*)
109+
dir_name="op_ut_with_skip_quantization_core";;
110+
*)
111+
dir_name=$(basename "$dir_path");;
112+
esac;
113+
mv "$1" "$dir_path/${dir_name}_$(basename "$1")"
114+
' _ {} \;
115+
cp op_ut_with_skip_nn/*.xml ${{ github.workspace }}/ut_log
116+
cp op_ut_with_skip_quantization/core/*.xml ${{ github.workspace }}/ut_log
117+
- command: 'python run_test_with_only.py'
118+
log_suffix: 'with_only'
119+
xml_post_processing: |
120+
cp op_ut_with_only.xml ${{ github.workspace }}/ut_log
121+
timeout: 10000
122+
env:
123+
PYTORCH_ENABLE_XPU_FALLBACK: '1'
124+
PYTORCH_TEST_WITH_SLOW: '1'
125+
- name: 'torch_xpu'
126+
condition: "contains(inputs.ut, 'torch_xpu')"
127+
directory: '../pytorch'
128+
command: |
129+
for xpu_case in build/bin/*{xpu,sycl}*; do
130+
if [[ "$xpu_case" != *"*"* && "$xpu_case" != *.so && "$xpu_case" != *.a ]]; then
131+
case_name=$(basename "$xpu_case")
132+
echo "Testing ${case_name} ..."
133+
"$xpu_case" --gtest_output=xml:"$TEST_REPORTS_DIR"/"$case_name".xml
134+
fi
135+
done
136+
test_cmd="python test/run_test.py --include "
137+
for test in $(ls test/inductor | grep test); do test_cmd="${test_cmd} inductor/$test"; done
138+
for test in $(ls test/xpu | grep test); do test_cmd="${test_cmd} xpu/$test"; done
139+
if [ -f "test/test_xpu.py" ]; then test_cmd="${test_cmd} test_xpu.py"; fi
140+
eval $test_cmd
141+
log_prefix: 'torch_xpu'
142+
timeout: 10000
143+
env:
144+
PYTORCH_TEST_WITH_SLOW: '1'
145+
PYTORCH_TESTING_DEVICE_ONLY_FOR: 'xpu'
146+
additional_steps: |
147+
TEST_REPORTS_DIR=${{ github.workspace }}/../pytorch/test/test-reports
148+
rm -rf "$TEST_REPORTS_DIR" && mkdir -p "$TEST_REPORTS_DIR"
149+
- name: 'xpu_profiling'
150+
condition: "${{ inputs.driver == 'rolling' && contains(inputs.ut, 'xpu_profiling') }}"
151+
directory: '${{ github.workspace }}'
152+
commands:
153+
- command: 'PROFILE=1 python -u test/profiling/rn50.py -a resnet50 --dummy ./ --num-iterations 20 --xpu 0'
154+
output_handling: |
155+
cp profiling.fp32.train.pt ${{ github.workspace }}/ut_log/profile_test
156+
log_file: 'rn50_profile.log'
157+
- command: 'python -u test/profiling/correlation_id_mixed.py'
158+
log_dir: 'issue_reproduce'
159+
log_file: 'correlation_id_mixed.log'
160+
- command: 'python -u test/profiling/reproducer.missing.gpu.kernel.time.py'
161+
log_dir: 'issue_reproduce'
162+
log_file: 'reproducer.missing.gpu.kernel.time.log'
163+
- command: 'python -u test/profiling/time_precision_in_profile.py'
164+
log_dir: 'issue_reproduce'
165+
log_file: 'time_precision_in_profile.log'
166+
- command: 'python -u test/profiling/profile_partial_runtime_ops.py'
167+
log_dir: 'issue_reproduce'
168+
log_file: 'profile_partial_runtime_ops.log'
169+
- command: 'python -u test/profiling/triton_xpu_ops_time.py'
170+
log_dir: 'issue_reproduce'
171+
log_file: 'triton_xpu_ops_time.log'
172+
- command: 'cd ../pytorch/test/profiler && python -m pytest --timeout 600 -vs test_cpp_thread.py'
173+
log_file: 'test_cpp_thread.log'
174+
- command: 'cd ../pytorch/test/profiler && python -m pytest --timeout 600 -vs test_execution_trace.py'
175+
log_file: 'test_execution_trace.log'
176+
- command: 'cd ../pytorch/test/profiler && python -m pytest --timeout 600 -vs test_memory_profiler.py'
177+
log_file: 'test_memory_profiler.log'
178+
- command: 'cd ../pytorch/test/profiler && python -m pytest --timeout 600 -vs test_profiler_tree.py'
179+
log_file: 'test_profiler_tree.log'
180+
timeout: 10000
181+
env: {}
182+
additional_steps: |
183+
mkdir -p ${{ github.workspace }}/ut_log/profile_test/issue_reproduce
54184
steps:
55185
- name: Checkout torch-xpu-ops
56186
uses: actions/checkout@v4
@@ -134,162 +264,48 @@ jobs:
134264
python pytorch/torch/utils/collect_env.py
135265
rm -rf /tmp/torchinductor_*
136266
rm -rf ~/.triton/cache
137-
- name: Run XPU OP Regression
138-
if: contains(inputs.ut, 'op_regression')
139-
run: |
140-
cd ${{ github.workspace }}
141-
mkdir -p ut_log/op_regression
142-
xpu-smi discovery
143-
source activate xpu_op_${ZE_AFFINITY_MASK}
144-
cd ${{ github.workspace }}
145-
cd test/regressions
146-
pip install pytest pytest-timeout
147-
timeout 8000 pytest --timeout 600 -v \
148-
--junit-xml=${{ github.workspace }}/ut_log/op_regression.xml \
149-
2>${{ github.workspace }}/ut_log/op_regression/op_regression_test_error.log | \
150-
tee ${{ github.workspace }}/ut_log/op_regression/op_regression_test.log
151-
- name: Run XPU OP Regressions test on device 1
152-
if: contains(inputs.ut, 'op_regression_dev1')
153-
run: |
154-
cd ${{ github.workspace }}
155-
mkdir -p ut_log/op_regression_dev1
156-
xpu-smi discovery
157-
source activate xpu_op_${ZE_AFFINITY_MASK}
158-
unset ZE_AFFINITY_MASK
159-
cd ${{ github.workspace }}
160-
cd test/regressions
161-
timeout 8000 pytest --timeout 600 -v test_operation_on_device_1.py \
162-
--junit-xml=${{ github.workspace }}/ut_log/op_regression_dev1.xml \
163-
2>${{ github.workspace }}/ut_log/op_regression_dev1/op_regression_dev1_test_error.log | \
164-
tee ${{ github.workspace }}/ut_log/op_regression_dev1/op_regression_dev1_test.log
165-
- name: Run XPU Transformers UT
166-
if: contains(inputs.ut, 'op_transformers')
167-
run: |
168-
source activate xpu_op_${ZE_AFFINITY_MASK}
169-
export PYTORCH_TEST_WITH_SLOW=1
170-
cd ${{ github.workspace }}
171-
mkdir -p ut_log/op_transformers
172-
cd ../pytorch/
173-
timeout 3600 pytest --timeout 600 -v test/test_transformers.py -k xpu \
174-
--junit-xml=${{ github.workspace }}/ut_log/op_transformers.xml \
175-
2>${{ github.workspace }}/ut_log/op_transformers/op_transformers_test_error.log | \
176-
tee ${{ github.workspace }}/ut_log/op_transformers/op_transformers_test.log
177-
- name: Run XPU OP Extended UT
178-
if: contains(inputs.ut, 'op_extended')
267+
- name: Run XPU UT Test
268+
if: ${{ matrix.test.condition }}
179269
run: |
180270
source activate xpu_op_${ZE_AFFINITY_MASK}
181-
export PYTORCH_TEST_WITH_SLOW=1
182271
cd ${{ github.workspace }}
183-
mkdir -p ut_log/op_extended
184-
cd ../pytorch/third_party/torch-xpu-ops/test/xpu/extended/
185-
timeout 10000 python run_test_with_skip.py \
186-
2>${{ github.workspace }}/ut_log/op_extended/op_extended_test_error.log | \
187-
tee ${{ github.workspace }}/ut_log/op_extended/op_extended_test.log
188-
cp op_extended.xml ${{ github.workspace }}/ut_log
189-
- name: Run XPU OP UT
190-
if: contains(inputs.ut, 'op_ut')
191-
run: |
192-
source activate xpu_op_${ZE_AFFINITY_MASK}
193-
export PYTORCH_ENABLE_XPU_FALLBACK=1
194-
export PYTORCH_TEST_WITH_SLOW=1
195-
cd ${{ github.workspace }}
196-
mkdir -p ut_log/op_ut
197-
cd ../pytorch/third_party/torch-xpu-ops/test/xpu
198-
timeout 10000 python run_test_with_skip.py \
199-
2>${{ github.workspace }}/ut_log/op_ut/op_ut_with_skip_test_error.log | \
200-
tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_skip_test.log
201-
cp *.xml ${{ github.workspace }}/ut_log
202-
find op_ut_with_skip_nn op_ut_with_skip_quantization/core -type f -exec sh -c '
203-
dir_path=$(dirname "$1");
204-
case "$dir_path" in
205-
*"op_ut_with_skip_quantization/core"*)
206-
dir_name="op_ut_with_skip_quantization_core";;
207-
*)
208-
dir_name=$(basename "$dir_path");;
209-
esac;
210-
mv "$1" "$dir_path/${dir_name}_$(basename "$1")"
211-
' _ {} \;
212-
cp op_ut_with_skip_nn/*.xml ${{ github.workspace }}/ut_log
213-
cp op_ut_with_skip_quantization/core/*.xml ${{ github.workspace }}/ut_log
214-
# Cases run with a on-demand white list, since some suites are too
215-
# slow to go through all operators on CPU. So add cases on-demand
216-
# when XPU implementatoin is done.
217-
# test_foreach, test_decomp
218-
timeout 10000 python run_test_with_only.py \
219-
2>${{ github.workspace }}/ut_log/op_ut/op_ut_with_only_test_error.log | \
220-
tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_only_test.log
221-
cp op_ut_with_only.xml ${{ github.workspace }}/ut_log
222-
- name: Run Torch XPU UT
223-
if: contains(inputs.ut, 'torch_xpu')
224-
run: |
225-
source activate xpu_op_${ZE_AFFINITY_MASK}
226-
cd ${{ github.workspace }}
227-
mkdir -p ut_log/torch_xpu
228-
cd ../pytorch
229-
TEST_REPORTS_DIR=$(pwd)/test/test-reports
230-
rm -rf "$TEST_REPORTS_DIR" && mkdir -p "$TEST_REPORTS_DIR"
231-
# Run Pytorch XPU binary UT
232-
for xpu_case in build/bin/*{xpu,sycl}*; do
233-
if [[ "$xpu_case" != *"*"* && "$xpu_case" != *.so && "$xpu_case" != *.a ]]; then
234-
case_name=$(basename "$xpu_case")
235-
echo "Testing ${case_name} ..."
236-
"$xpu_case" --gtest_output=xml:"$TEST_REPORTS_DIR"/"$case_name".xml \
237-
2>${{ github.workspace }}/ut_log/torch_xpu/binary_ut_torch_xpu_${case_name}_test_error.log | \
238-
tee ${{ github.workspace }}/ut_log/torch_xpu/binary_ut_torch_xpu_${case_name}_test.log
239-
fi
240-
done
241-
# Run Pytorch XPU python UT
242-
export PYTORCH_TEST_WITH_SLOW=1
243-
export PYTORCH_TESTING_DEVICE_ONLY_FOR="xpu"
272+
mkdir -p ut_log/${{ matrix.test.log_prefix }}
273+
${{ matrix.test.additional_steps }}
244274
245-
test_cmd="python test/run_test.py --include "
246-
# All Inductor UT under test/inductor
247-
for test in $(ls test/inductor | grep test);
248-
do
249-
test_cmd="${test_cmd} inductor/$test";
250-
done
251-
# All xpu ut under test/xpu
252-
for test in $(ls test/xpu | grep test);
253-
do
254-
test_cmd="${test_cmd} xpu/$test";
255-
done
256-
if [ -f "test/test_xpu.py" ]; then
257-
test_cmd="${test_cmd} test_xpu.py"
258-
fi
259-
eval $test_cmd 2>${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_test_error.log | \
260-
tee ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_test.log
261-
- name: Run Torch XPU Profile UT
262-
if: ${{ inputs.driver == 'rolling' && contains(inputs.ut, 'xpu_profiling') }}
263-
run: |
264-
source activate xpu_op_${ZE_AFFINITY_MASK}
265-
mkdir -p ${{ github.workspace }}/ut_log/profile_test
266-
# RN50 Test
267-
PROFILE=1 python -u test/profiling/rn50.py -a resnet50 --dummy ./ --num-iterations 20 --xpu 0
268-
cp profiling.fp32.train.pt ${{ github.workspace }}/ut_log/profile_test
269-
270-
# All Issue Reproduce UT
271-
mkdir -p ${{ github.workspace }}/ut_log/profile_test/issue_reproduce
272-
python -u test/profiling/correlation_id_mixed.py | \
273-
tee ${{ github.workspace }}/ut_log/profile_test/issue_reproduce/correlation_id_mixed.log
274-
python -u test/profiling/reproducer.missing.gpu.kernel.time.py | \
275-
tee ${{ github.workspace }}/ut_log/profile_test/issue_reproduce/reproducer.missing.gpu.kernel.time.log
276-
python -u test/profiling/time_precision_in_profile.py | \
277-
tee ${{ github.workspace }}/ut_log/profile_test/issue_reproduce/time_precision_in_profile.log
278-
python -u test/profiling/profile_partial_runtime_ops.py | \
279-
tee ${{ github.workspace }}/ut_log/profile_test/issue_reproduce/profile_partial_runtime_ops.log
280-
python -u test/profiling/triton_xpu_ops_time.py | \
281-
tee ${{ github.workspace }}/ut_log/profile_test/issue_reproduce/triton_xpu_ops_time.log
275+
export ${{ join(matrix.test.env.*, ' ') }}
276+
cd ${{ matrix.test.directory }}
282277
283-
# All xpu ut under test/profiler
284-
cd ../pytorch/test/profiler
285-
python -m pytest --timeout 600 -vs test_cpp_thread.py | \
286-
tee ${{ github.workspace }}/ut_log/profile_test/test_cpp_thread.log
287-
python -m pytest --timeout 600 -vs test_execution_trace.py | \
288-
tee ${{ github.workspace }}/ut_log/profile_test/test_execution_trace.log
289-
python -m pytest --timeout 600 -vs test_memory_profiler.py | \
290-
tee ${{ github.workspace }}/ut_log/profile_test/test_memory_profiler.log
291-
python -m pytest --timeout 600 -vs test_profiler_tree.py | \
292-
tee ${{ github.workspace }}/ut_log/profile_test/test_profiler_tree.log
278+
if [ "${{ matrix.test.name }}" == "xpu_profiling" ]; then
279+
${{ each command in matrix.test.commands }}
280+
echo "Running profile command: ${{ command.command }}"
281+
if [[ "${{ command.command }}" == *"cd "* ]]; then
282+
( ${{ command.command }} ) | \
283+
tee ${{ github.workspace }}/ut_log/profile_test/${{ command.log_dir || '' }}/${{ command.log_file }}
284+
else
285+
${{ command.command }} | \
286+
tee ${{ github.workspace }}/ut_log/profile_test/${{ command.log_dir || '' }}/${{ command.log_file }}
287+
fi
288+
${{ command.output_handling || '' }}
289+
${{ end }}
290+
else
291+
if [ -z "${{ matrix.test.commands }}" ]; then
292+
echo "Running single command test"
293+
timeout ${{ matrix.test.timeout }} ${{ matrix.test.command }} \
294+
2>${{ github.workspace }}/ut_log/${{ matrix.test.name }}/${{ matrix.test.log_prefix }}_test_error.log | \
295+
tee ${{ github.workspace }}/ut_log/${{ matrix.test.name }}/${{ matrix.test.log_suffix }}_test.log
296+
297+
${{ matrix.test.xml_post_processing || '' }}
298+
else
299+
echo "Running multi-command test"
300+
${{ each command in matrix.test.commands }}
301+
echo "Running command: ${{ command.command }}"
302+
timeout ${{ matrix.test.timeout }} ${{ command.command }} \
303+
2>${{ github.workspace }}/ut_log/${{ matrix.test.name }}/${{ matrix.test.log_prefix }}_${{ command.log_suffix }}_test_error.log | \
304+
tee ${{ github.workspace }}/ut_log/${{ matrix.test.name }}/${{ matrix.test.log_prefix }}_${{ command.log_suffix }}_test.log
305+
306+
${{ command.xml_post_processing || '' }}
307+
${{ end }}
308+
fi
293309
- name: UT Test Results Summary
294310
run: |
295311
source activate xpu_op_${ZE_AFFINITY_MASK}

0 commit comments

Comments
 (0)