From 9df9f1c221de984b206fe3f56e1f8da7fe2276ec Mon Sep 17 00:00:00 2001 From: mengfeil Date: Tue, 18 Feb 2025 13:29:38 +0800 Subject: [PATCH 1/9] modify accuracy detail --- .github/scripts/e2e_summary.sh | 50 ++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/.github/scripts/e2e_summary.sh b/.github/scripts/e2e_summary.sh index 89f11f800..9c22a554e 100644 --- a/.github/scripts/e2e_summary.sh +++ b/.github/scripts/e2e_summary.sh @@ -3,6 +3,49 @@ results_dir="$1" check_file="$(dirname "$0")/../ci_expected_accuracy/check_expected.py" +function get_model_result() { + echo -e "\n + + + " + suite_list=$( + find "${results_dir}" -name "*.csv" |grep -E "_xpu_accuracy.csv" |\ + sed "s/.*inductor_//;s/_[abf].*//" |sort |uniq + ) + for suite in ${suite_list} + do + model_list=($( + find "${results_dir}" -name "*.csv" |grep -E ".*${suite}.*_xpu_accuracy.csv" |\ + xargs cat |grep "^xpu," |cut -d, -f2 |sort |uniq + )) + for model in ${model_list[*]} + do + for dtype in float32 bfloat16 float16 amp_bf16 amp_fp16 + do + for mode in training inference + do + eval "${mode}_${dtype}=\$(find "${results_dir}" -name "*.csv" |grep -E ".*${suite}_${dtype}_${mode}_xpu_accuracy.csv" |xargs grep ",${model}," |cut -d, -f4)" + done + done + echo -e " + + + + + + + + + + + + + " + done + done + echo -e "
Suite Model Training Inference
float32 bfloat16 float16 amp_bf16 amp_fp16 float32 bfloat16 float16 amp_bf16 amp_fp16
${suite}${model}${training_float32}${training_bfloat16}${training_float16}${training_amp_bf16}${training_amp_fp16}${inference_float32}${inference_bfloat16}${inference_float16}${inference_amp_bf16}${inference_amp_fp16}
\n" +} + # Accuracy accuracy=$(find "${results_dir}" -name "*.csv" |grep -E "_xpu_accuracy.csv" -c) if [ "${accuracy}" -gt 0 ];then @@ -64,14 +107,9 @@ if [ "${accuracy}" -gt 0 ];then s/Warning timeout models:/$${\\color{orange}Warning \\space timeout \\space models}$$:/g; s/Failed to passed models:/$${\\color{green}Failed \\space to \\space passed \\space models}$$:/g; ' tmp-result.txt - { - echo "" - sed "1d" tmp-result.txt |awk -F: '{printf("\n", $1, $2)}' - echo -e "
$(sed 's/=//g' tmp-result.txt |head -n 1)
%s%s
\n" - } >> tmp-details.txt done cat tmp-summary.txt - grep -v " 0 \[\]" tmp-details.txt + get_model_result rm -rf tmp-*.txt fi From 522eb79059d24e0d56c94b17bf029c61d6a13d70 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Tue, 18 Feb 2025 15:17:53 +0800 Subject: [PATCH 2/9] colorful --- .github/scripts/e2e_summary.sh | 49 +++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/.github/scripts/e2e_summary.sh b/.github/scripts/e2e_summary.sh index 9c22a554e..d1b8e714a 100644 --- a/.github/scripts/e2e_summary.sh +++ b/.github/scripts/e2e_summary.sh @@ -5,8 +5,13 @@ check_file="$(dirname "$0")/../ci_expected_accuracy/check_expected.py" function get_model_result() { echo -e "\n - - + + + + + + + " suite_list=$( find "${results_dir}" -name "*.csv" |grep -E "_xpu_accuracy.csv" |\ @@ -18,13 +23,39 @@ function get_model_result() { find "${results_dir}" -name "*.csv" |grep -E ".*${suite}.*_xpu_accuracy.csv" |\ xargs cat |grep "^xpu," |cut -d, -f2 |sort |uniq )) + for dtype in float32 bfloat16 float16 amp_bf16 amp_fp16 + do + for mode in training inference + do + python "${check_file}" --suite "${suite}" --mode "${mode}" --dtype "${dtype}" \ + --csv_file "$(find "${results_dir}" -name "*.csv" |\ + grep -E ".*${suite}_${dtype}_${mode}_xpu_accuracy.csv" + )" > tmp-${dtype}-${mode}.txt + done + done for model in ${model_list[*]} do for dtype in float32 bfloat16 float16 amp_bf16 amp_fp16 do for mode in training inference do - eval "${mode}_${dtype}=\$(find "${results_dir}" -name "*.csv" |grep -E ".*${suite}_${dtype}_${mode}_xpu_accuracy.csv" |xargs grep ",${model}," |cut -d, -f4)" + colorful=$(grep "${model}" tmp-${dtype}-${mode}.txt |awk 'BEGIN{color="black";}{ + if ($0 ~/Real failed/){ + color="red"; + }else if ($0 ~/Expected failed/){ + color="blue"; + }else if ($0 ~/Warning timeout/){ + color="orange"; + }else if ($0 ~/New models/){ + color="blue"; + }else if ($0 ~/Failed to passed/){ + color="green"; + } + }END{print color;}') + context=$(find "${results_dir}" -name "*.csv" |\ + grep -E ".*${suite}_${dtype}_${mode}_xpu_accuracy.csv" |xargs grep ",${model}," |cut -d, -f4 |\ + awk -v c="${colorful}" '{if(c=="black") {print $0}else {printf("\\$\\${__color__{%s}%s}\\$\\$", c, $0)}}') + eval "${mode}_${dtype}=${context}" done done echo -e " @@ -40,7 +71,7 @@ function get_model_result() { - " + " |sed '/__color__/{s/__color__/\\color/g;s/_/\\_/g}' done done echo -e "
Suite Model Training Inference
float32 bfloat16 float16 amp_bf16 amp_fp16 float32 bfloat16 float16 amp_bf16 amp_fp16
Suite Model Training Inference
float32 bfloat16 float16 amp_bf16 amp_fp16 float32 bfloat16 float16 amp_bf16 amp_fp16
${inference_float16} ${inference_amp_bf16} ${inference_amp_fp16}
\n" @@ -49,9 +80,15 @@ function get_model_result() { # Accuracy accuracy=$(find "${results_dir}" -name "*.csv" |grep -E "_xpu_accuracy.csv" -c) if [ "${accuracy}" -gt 0 ];then + printf "Note: +\$\${\\color{red}Red}\$\$: the failed cases which need look into +\$\${\\color{green}Green}\$\$: the new passed cases which need update reference +\$\${\\color{blue}Blue}\$\$: the expected failed or new enabled cases +\$\${\\color{orange}Orange}\$\$: the warning cases\n\n" echo "### Accuracy" - printf "| Category | Total | \$\${\\color{green}Passed}\$\$ | Pass Rate | \$\${\\color{red}Failed}\$\$ | " - printf "\$\${\\color{blue}Xfailed}\$\$ | \$\${\\color{orange}Timeout}\$\$ | New Passed | New Enabled | Not Run |\n" + printf "| Category | Total | Passed | Pass Rate | \$\${\\color{red}Failed}\$\$ | " + printf "\$\${\\color{blue}Xfailed}\$\$ | \$\${\\color{orange}Timeout}\$\$ | " + printf "\$\${\\color{green}New Passed}\$\$ | \$\${\\color{blue}New Enabled}\$\$ | Not Run |\n" printf "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" echo > tmp-summary.txt echo > tmp-details.txt From a8945302a184b607f0c677747ddccda55d653bd0 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Tue, 18 Feb 2025 15:58:30 +0800 Subject: [PATCH 3/9] update --- .github/scripts/e2e_summary.sh | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/.github/scripts/e2e_summary.sh b/.github/scripts/e2e_summary.sh index d1b8e714a..abe9164bb 100644 --- a/.github/scripts/e2e_summary.sh +++ b/.github/scripts/e2e_summary.sh @@ -23,23 +23,13 @@ function get_model_result() { find "${results_dir}" -name "*.csv" |grep -E ".*${suite}.*_xpu_accuracy.csv" |\ xargs cat |grep "^xpu," |cut -d, -f2 |sort |uniq )) - for dtype in float32 bfloat16 float16 amp_bf16 amp_fp16 - do - for mode in training inference - do - python "${check_file}" --suite "${suite}" --mode "${mode}" --dtype "${dtype}" \ - --csv_file "$(find "${results_dir}" -name "*.csv" |\ - grep -E ".*${suite}_${dtype}_${mode}_xpu_accuracy.csv" - )" > tmp-${dtype}-${mode}.txt - done - done for model in ${model_list[*]} do for dtype in float32 bfloat16 float16 amp_bf16 amp_fp16 do for mode in training inference do - colorful=$(grep "${model}" tmp-${dtype}-${mode}.txt |awk 'BEGIN{color="black";}{ + colorful=$(grep "${model}" tmp-${suite}-${mode}-${dtype}.txt 2>&1 |awk 'BEGIN{color="black";}{ if ($0 ~/Real failed/){ color="red"; }else if ($0 ~/Expected failed/){ @@ -97,9 +87,9 @@ if [ "${accuracy}" -gt 0 ];then category="$(echo "${csv}" |sed 's/.*inductor_//;s/_xpu_accuracy.*//')" suite="$(echo "${csv}" |sed 's/.*inductor_//;s/_.*//;s/timm/timm_models/')" mode="$(echo "${csv}" |sed 's/_xpu_accuracy.*//;s/.*_//')" - dt="$(echo "${csv}" |sed -E 's/.*inductor_[a-z]*_//;s/models_//;s/_infer.*|_train.*//')" - python "${check_file}" --suite "${suite}" --mode "${mode}" --dtype "${dt}" --csv_file "${csv}" > tmp-result.txt - test_result="$(sed 's/, /,/g' tmp-result.txt |awk '{ + dtype="$(echo "${csv}" |sed -E 's/.*inductor_[a-z]*_//;s/models_//;s/_infer.*|_train.*//')" + python "${check_file}" --suite "${suite}" --mode "${mode}" --dtype "${dtype}" --csv_file "${csv}" > tmp-${suite}-${mode}-${dtype}.txt + test_result="$(sed 's/, /,/g' tmp-${suite}-${mode}-${dtype}.txt |awk '{ if($0 ~/Total/){ total = $3; } @@ -138,12 +128,6 @@ if [ "${accuracy}" -gt 0 ];then total, passed, pass_rate, failed, xfail, timeout, new_passed, new_enabled, not_run); }')" echo "| ${category} | ${test_result} |" >> tmp-summary.txt - sed -i ' - s/Real failed models:/$${\\color{red}Real \\space failed \\space models}$$:/g; - s/Expected failed models:/$${\\color{blue}Expected \\space failed \\space models}$$:/g; - s/Warning timeout models:/$${\\color{orange}Warning \\space timeout \\space models}$$:/g; - s/Failed to passed models:/$${\\color{green}Failed \\space to \\space passed \\space models}$$:/g; - ' tmp-result.txt done cat tmp-summary.txt get_model_result From 4ded008ebf44a01ba2870d9efc023bd344f70ffb Mon Sep 17 00:00:00 2001 From: mengfeil Date: Tue, 18 Feb 2025 16:39:33 +0800 Subject: [PATCH 4/9] ignore shell check SC1083 --- .github/scripts/e2e_summary.sh | 33 +++++++++++++++++---------------- .github/scripts/lintrunner.sh | 3 +++ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.github/scripts/e2e_summary.sh b/.github/scripts/e2e_summary.sh index abe9164bb..62d83d165 100644 --- a/.github/scripts/e2e_summary.sh +++ b/.github/scripts/e2e_summary.sh @@ -19,17 +19,17 @@ function get_model_result() { ) for suite in ${suite_list} do - model_list=($( + model_list=$( find "${results_dir}" -name "*.csv" |grep -E ".*${suite}.*_xpu_accuracy.csv" |\ xargs cat |grep "^xpu," |cut -d, -f2 |sort |uniq - )) - for model in ${model_list[*]} + ) + for model in ${model_list} do for dtype in float32 bfloat16 float16 amp_bf16 amp_fp16 do for mode in training inference do - colorful=$(grep "${model}" tmp-${suite}-${mode}-${dtype}.txt 2>&1 |awk 'BEGIN{color="black";}{ + colorful=$(grep "${model}" "tmp-${suite}-${mode}-${dtype}.txt" 2>&1 |awk 'BEGIN{color="black";}{ if ($0 ~/Real failed/){ color="red"; }else if ($0 ~/Expected failed/){ @@ -51,16 +51,17 @@ function get_model_result() { echo -e " ${suite} ${model} - ${training_float32} - ${training_bfloat16} - ${training_float16} - ${training_amp_bf16} - ${training_amp_fp16} - ${inference_float32} - ${inference_bfloat16} - ${inference_float16} - ${inference_amp_bf16} - ${inference_amp_fp16} + $(eval " + for mode in training inference + do + for dtype in float32 bfloat16 float16 amp_bf16 amp_fp16 + do + printf '' + printf "\${${mode}_${dtype}}" + printf '' + done + done + ") " |sed '/__color__/{s/__color__/\\color/g;s/_/\\_/g}' done done @@ -88,8 +89,8 @@ if [ "${accuracy}" -gt 0 ];then suite="$(echo "${csv}" |sed 's/.*inductor_//;s/_.*//;s/timm/timm_models/')" mode="$(echo "${csv}" |sed 's/_xpu_accuracy.*//;s/.*_//')" dtype="$(echo "${csv}" |sed -E 's/.*inductor_[a-z]*_//;s/models_//;s/_infer.*|_train.*//')" - python "${check_file}" --suite "${suite}" --mode "${mode}" --dtype "${dtype}" --csv_file "${csv}" > tmp-${suite}-${mode}-${dtype}.txt - test_result="$(sed 's/, /,/g' tmp-${suite}-${mode}-${dtype}.txt |awk '{ + python "${check_file}" --suite "${suite}" --mode "${mode}" --dtype "${dtype}" --csv_file "${csv}" > "tmp-${suite}-${mode}-${dtype}.txt" + test_result="$(sed 's/, /,/g' "tmp-${suite}-${mode}-${dtype}.txt" |awk '{ if($0 ~/Total/){ total = $3; } diff --git a/.github/scripts/lintrunner.sh b/.github/scripts/lintrunner.sh index df21529ad..ecac860a9 100755 --- a/.github/scripts/lintrunner.sh +++ b/.github/scripts/lintrunner.sh @@ -23,6 +23,9 @@ if ! command -v lintrunner &> /dev/null; then python3 -m pip install lintrunner==0.12.7 fi +# Ignoring errors in one specific run +export SHELLCHECK_OPTS="-e SC1083" + # This has already been cached in the docker image lintrunner init 2> /dev/null From 606c40c26b6f75a617baf1b4f6977327b276a2f4 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Tue, 18 Feb 2025 16:46:03 +0800 Subject: [PATCH 5/9] update empty comment --- .github/scripts/e2e_summary.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/e2e_summary.sh b/.github/scripts/e2e_summary.sh index 62d83d165..33ab72dd2 100644 --- a/.github/scripts/e2e_summary.sh +++ b/.github/scripts/e2e_summary.sh @@ -71,11 +71,12 @@ function get_model_result() { # Accuracy accuracy=$(find "${results_dir}" -name "*.csv" |grep -E "_xpu_accuracy.csv" -c) if [ "${accuracy}" -gt 0 ];then - printf "Note: + printf "#### Note: \$\${\\color{red}Red}\$\$: the failed cases which need look into \$\${\\color{green}Green}\$\$: the new passed cases which need update reference \$\${\\color{blue}Blue}\$\$: the expected failed or new enabled cases -\$\${\\color{orange}Orange}\$\$: the warning cases\n\n" +\$\${\\color{orange}Orange}\$\$: the warning cases +Empty means the cases NOT run\n\n" echo "### Accuracy" printf "| Category | Total | Passed | Pass Rate | \$\${\\color{red}Failed}\$\$ | " printf "\$\${\\color{blue}Xfailed}\$\$ | \$\${\\color{orange}Timeout}\$\$ | " From c60695dd8eafdb0bc3bbd5bc2ed92b3f7d6222e8 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Tue, 18 Feb 2025 18:36:47 +0800 Subject: [PATCH 6/9] Update failed condition --- .github/scripts/e2e_summary.sh | 14 ++++++++++---- .github/workflows/nightly_ondemand.yml | 16 +++++----------- .github/workflows/nightly_ondemand_rolling.yml | 16 +++++----------- .github/workflows/nightly_ondemand_whl.yml | 16 +++++----------- .github/workflows/pull.yml | 9 +++++---- 5 files changed, 30 insertions(+), 41 deletions(-) diff --git a/.github/scripts/e2e_summary.sh b/.github/scripts/e2e_summary.sh index 33ab72dd2..a239db4e8 100644 --- a/.github/scripts/e2e_summary.sh +++ b/.github/scripts/e2e_summary.sh @@ -17,6 +17,7 @@ function get_model_result() { find "${results_dir}" -name "*.csv" |grep -E "_xpu_accuracy.csv" |\ sed "s/.*inductor_//;s/_[abf].*//" |sort |uniq ) + rm -rf /tmp/tmp-result.txt for suite in ${suite_list} do model_list=$( @@ -29,9 +30,13 @@ function get_model_result() { do for mode in training inference do - colorful=$(grep "${model}" "tmp-${suite}-${mode}-${dtype}.txt" 2>&1 |awk 'BEGIN{color="black";}{ + colorful=$(grep "${model}" "tmp-${suite}-${mode}-${dtype}.txt" 2>&1 |awk 'BEGIN{ + color = "black"; + exit_label = 0; + }{ if ($0 ~/Real failed/){ color="red"; + exit_label++; }else if ($0 ~/Expected failed/){ color="blue"; }else if ($0 ~/Warning timeout/){ @@ -40,11 +45,13 @@ function get_model_result() { color="blue"; }else if ($0 ~/Failed to passed/){ color="green"; + exit_label++; } - }END{print color;}') + }END{print color, exit_label}') + echo "${colorful}" >> /tmp/tmp-result.txt context=$(find "${results_dir}" -name "*.csv" |\ grep -E ".*${suite}_${dtype}_${mode}_xpu_accuracy.csv" |xargs grep ",${model}," |cut -d, -f4 |\ - awk -v c="${colorful}" '{if(c=="black") {print $0}else {printf("\\$\\${__color__{%s}%s}\\$\\$", c, $0)}}') + awk -v c="${colorful/ *}" '{if(c=="black") {print $0}else {printf("\\$\\${__color__{%s}%s}\\$\\$", c, $0)}}') eval "${mode}_${dtype}=${context}" done done @@ -133,7 +140,6 @@ Empty means the cases NOT run\n\n" done cat tmp-summary.txt get_model_result - rm -rf tmp-*.txt fi # Performance diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml index eee050ddb..69cd51a4c 100644 --- a/.github/workflows/nightly_ondemand.yml +++ b/.github/workflows/nightly_ondemand.yml @@ -287,18 +287,12 @@ jobs: tar zcf xpu-inductor-${GITHUB_RUN_ID}.tgz -C ${{ github.workspace }}/upload_files/ . # backup logs # Print summary bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} - failed_models=$(grep "Real failed models: *[1-9]" ${GITHUB_STEP_SUMMARY} |wc -l || true) - timeout_models=$(grep "timeout models: *[1-9]" ${GITHUB_STEP_SUMMARY} |wc -l || true) - if [ ${timeout_models} -ne 0 ];then - TIMEOUT_MODELS="$( - grep -B 1 "timeout models: [1-9]" ${GITHUB_STEP_SUMMARY} - )" - echo "TIMEOUT_MODELS=\"${TIMEOUT_MODELS}\"" |awk '{printf("%s\\n", $0)}' |sed 's/\\n$//' |tee -a "${GITHUB_OUTPUT}" - fi - if [ ${failed_models} -ne 0 ];then - grep -E "Real failed models: [1-9]|Summary for" ${GITHUB_STEP_SUMMARY} |grep "failed" -B 1 - exit 1 + exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" tmp-*.txt |grep -E "failed|passed|timeout" -B 1 + if [ ${exit_label} -ne 0 ];then + echo "There are ${exit_label} cases that need look into!!! Please check them" fi + exit ${exit_label} - name: Upload Inductor XPU E2E Data if: ${{ ! cancelled() }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/nightly_ondemand_rolling.yml b/.github/workflows/nightly_ondemand_rolling.yml index 1e264f5b4..6db333fac 100644 --- a/.github/workflows/nightly_ondemand_rolling.yml +++ b/.github/workflows/nightly_ondemand_rolling.yml @@ -298,18 +298,12 @@ jobs: tar zcf xpu-inductor-${GITHUB_RUN_ID}.tgz -C ${{ github.workspace }}/upload_files/ . # backup logs # Print summary bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} - failed_models=$(grep "Real failed models: *[1-9]" ${GITHUB_STEP_SUMMARY} |wc -l || true) - timeout_models=$(grep "timeout models: *[1-9]" ${GITHUB_STEP_SUMMARY} |wc -l || true) - if [ ${timeout_models} -ne 0 ];then - TIMEOUT_MODELS="$( - grep -B 1 "timeout models: [1-9]" ${GITHUB_STEP_SUMMARY} - )" - echo "TIMEOUT_MODELS=\"${TIMEOUT_MODELS}\"" |awk '{printf("%s\\n", $0)}' |sed 's/\\n$//' |tee -a "${GITHUB_OUTPUT}" - fi - if [ ${failed_models} -ne 0 ];then - grep -E "Real failed models: [1-9]|Summary for" ${GITHUB_STEP_SUMMARY} |grep "failed" -B 1 - exit 1 + exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" tmp-*.txt |grep -E "failed|passed|timeout" -B 1 + if [ ${exit_label} -ne 0 ];then + echo "There are ${exit_label} cases that need look into!!! Please check them" fi + exit ${exit_label} - name: Upload Inductor XPU E2E Data if: ${{ ! cancelled() }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/nightly_ondemand_whl.yml b/.github/workflows/nightly_ondemand_whl.yml index 8a55fb4c9..35a4dc38d 100644 --- a/.github/workflows/nightly_ondemand_whl.yml +++ b/.github/workflows/nightly_ondemand_whl.yml @@ -243,18 +243,12 @@ jobs: tar zcf xpu-inductor-${GITHUB_RUN_ID}.tgz -C ${{ github.workspace }}/upload_files/ . # backup logs # Print summary bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} - failed_models=$(grep "Real failed models: *[1-9]" ${GITHUB_STEP_SUMMARY} |wc -l || true) - timeout_models=$(grep "timeout models: *[1-9]" ${GITHUB_STEP_SUMMARY} |wc -l || true) - if [ ${timeout_models} -ne 0 ];then - TIMEOUT_MODELS="$( - grep -B 1 "timeout models: [1-9]" ${GITHUB_STEP_SUMMARY} - )" - echo "TIMEOUT_MODELS=\"${TIMEOUT_MODELS}\"" |awk '{printf("%s\\n", $0)}' |sed 's/\\n$//' |tee -a "${GITHUB_OUTPUT}" - fi - if [ ${failed_models} -ne 0 ];then - grep -E "Real failed models: [1-9]|Summary for" ${GITHUB_STEP_SUMMARY} |grep "failed" -B 1 - exit 1 + exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" tmp-*.txt |grep -E "failed|passed|timeout" -B 1 + if [ ${exit_label} -ne 0 ];then + echo "There are ${exit_label} cases that need look into!!! Please check them" fi + exit ${exit_label} - name: Upload Inductor XPU E2E Data if: ${{ ! cancelled() }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 09d09af5e..48783856d 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -195,11 +195,12 @@ jobs: cp -r ${{ github.workspace }}/../pytorch/inductor_log ${{ github.workspace }}/upload_files # Print summary bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} - failed_models=$(grep "Real failed models: *[1-9]" ${GITHUB_STEP_SUMMARY} |wc -l || true) - if [ ${failed_models} -ne 0 ];then - grep -E "Real failed models: [1-9]|Summary for" ${GITHUB_STEP_SUMMARY} |grep "failed" -B 1 - exit 1 + exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" tmp-*.txt |grep -E "failed|passed|timeout" -B 1 + if [ ${exit_label} -ne 0 ];then + echo "There are ${exit_label} cases that need look into!!! Please check them" fi + exit ${exit_label} - name: Upload Inductor XPU E2E Data if: ${{ ! cancelled() }} uses: actions/upload-artifact@v4 From c19839602383dbbf2681e3b929a10425900cc34a Mon Sep 17 00:00:00 2001 From: mengfeil Date: Wed, 19 Feb 2025 10:19:05 +0800 Subject: [PATCH 7/9] update --- .github/scripts/e2e_summary.sh | 23 +++++++++++------------ .github/scripts/lintrunner.sh | 2 +- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/scripts/e2e_summary.sh b/.github/scripts/e2e_summary.sh index a239db4e8..370015f95 100644 --- a/.github/scripts/e2e_summary.sh +++ b/.github/scripts/e2e_summary.sh @@ -52,23 +52,22 @@ function get_model_result() { context=$(find "${results_dir}" -name "*.csv" |\ grep -E ".*${suite}_${dtype}_${mode}_xpu_accuracy.csv" |xargs grep ",${model}," |cut -d, -f4 |\ awk -v c="${colorful/ *}" '{if(c=="black") {print $0}else {printf("\\$\\${__color__{%s}%s}\\$\\$", c, $0)}}') - eval "${mode}_${dtype}=${context}" + eval "export ${mode}_${dtype}=${context}" done done echo -e " ${suite} ${model} - $(eval " - for mode in training inference - do - for dtype in float32 bfloat16 float16 amp_bf16 amp_fp16 - do - printf '' - printf "\${${mode}_${dtype}}" - printf '' - done - done - ") + ${training_float32} + ${training_bfloat16} + ${training_float16} + ${training_amp_bf16} + ${training_amp_fp16} + ${inference_float32} + ${inference_bfloat16} + ${inference_float16} + ${inference_amp_bf16} + ${inference_amp_fp16} " |sed '/__color__/{s/__color__/\\color/g;s/_/\\_/g}' done done diff --git a/.github/scripts/lintrunner.sh b/.github/scripts/lintrunner.sh index ecac860a9..1956de5dc 100755 --- a/.github/scripts/lintrunner.sh +++ b/.github/scripts/lintrunner.sh @@ -24,7 +24,7 @@ if ! command -v lintrunner &> /dev/null; then fi # Ignoring errors in one specific run -export SHELLCHECK_OPTS="-e SC1083" +export SHELLCHECK_OPTS="-e SC2154" # This has already been cached in the docker image lintrunner init 2> /dev/null From 8133e45ca2772b483399bd57825394101f5fd7a6 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Wed, 19 Feb 2025 13:08:20 +0800 Subject: [PATCH 8/9] modify tmp files path --- .github/scripts/e2e_summary.sh | 14 +++++++------- .github/workflows/nightly_ondemand.yml | 2 +- .github/workflows/nightly_ondemand_rolling.yml | 2 +- .github/workflows/nightly_ondemand_whl.yml | 2 +- .github/workflows/pull.yml | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/scripts/e2e_summary.sh b/.github/scripts/e2e_summary.sh index 370015f95..47e1b4b4d 100644 --- a/.github/scripts/e2e_summary.sh +++ b/.github/scripts/e2e_summary.sh @@ -30,7 +30,7 @@ function get_model_result() { do for mode in training inference do - colorful=$(grep "${model}" "tmp-${suite}-${mode}-${dtype}.txt" 2>&1 |awk 'BEGIN{ + colorful=$(grep "${model}" "/tmp/tmp-${suite}-${mode}-${dtype}.txt" 2>&1 |awk 'BEGIN{ color = "black"; exit_label = 0; }{ @@ -88,16 +88,16 @@ Empty means the cases NOT run\n\n" printf "\$\${\\color{blue}Xfailed}\$\$ | \$\${\\color{orange}Timeout}\$\$ | " printf "\$\${\\color{green}New Passed}\$\$ | \$\${\\color{blue}New Enabled}\$\$ | Not Run |\n" printf "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" - echo > tmp-summary.txt - echo > tmp-details.txt + echo > /tmp/tmp-summary.txt + echo > /tmp/tmp-details.txt for csv in $(find "${results_dir}" -name "*.csv" |grep -E "_xpu_accuracy.csv" |sort) do category="$(echo "${csv}" |sed 's/.*inductor_//;s/_xpu_accuracy.*//')" suite="$(echo "${csv}" |sed 's/.*inductor_//;s/_.*//;s/timm/timm_models/')" mode="$(echo "${csv}" |sed 's/_xpu_accuracy.*//;s/.*_//')" dtype="$(echo "${csv}" |sed -E 's/.*inductor_[a-z]*_//;s/models_//;s/_infer.*|_train.*//')" - python "${check_file}" --suite "${suite}" --mode "${mode}" --dtype "${dtype}" --csv_file "${csv}" > "tmp-${suite}-${mode}-${dtype}.txt" - test_result="$(sed 's/, /,/g' "tmp-${suite}-${mode}-${dtype}.txt" |awk '{ + python "${check_file}" --suite "${suite}" --mode "${mode}" --dtype "${dtype}" --csv_file "${csv}" > "/tmp/tmp-${suite}-${mode}-${dtype}.txt" + test_result="$(sed 's/, /,/g' "/tmp/tmp-${suite}-${mode}-${dtype}.txt" |awk '{ if($0 ~/Total/){ total = $3; } @@ -135,9 +135,9 @@ Empty means the cases NOT run\n\n" printf(" %d | %d | %s | %d | %d | %d | %d | %d | %d\n", total, passed, pass_rate, failed, xfail, timeout, new_passed, new_enabled, not_run); }')" - echo "| ${category} | ${test_result} |" >> tmp-summary.txt + echo "| ${category} | ${test_result} |" >> /tmp/tmp-summary.txt done - cat tmp-summary.txt + cat /tmp/tmp-summary.txt get_model_result fi diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml index 69cd51a4c..5866adb73 100644 --- a/.github/workflows/nightly_ondemand.yml +++ b/.github/workflows/nightly_ondemand.yml @@ -288,7 +288,7 @@ jobs: # Print summary bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) - grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" tmp-*.txt |grep -E "failed|passed|timeout" -B 1 + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 if [ ${exit_label} -ne 0 ];then echo "There are ${exit_label} cases that need look into!!! Please check them" fi diff --git a/.github/workflows/nightly_ondemand_rolling.yml b/.github/workflows/nightly_ondemand_rolling.yml index 6db333fac..34f834e3c 100644 --- a/.github/workflows/nightly_ondemand_rolling.yml +++ b/.github/workflows/nightly_ondemand_rolling.yml @@ -299,7 +299,7 @@ jobs: # Print summary bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) - grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" tmp-*.txt |grep -E "failed|passed|timeout" -B 1 + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 if [ ${exit_label} -ne 0 ];then echo "There are ${exit_label} cases that need look into!!! Please check them" fi diff --git a/.github/workflows/nightly_ondemand_whl.yml b/.github/workflows/nightly_ondemand_whl.yml index 35a4dc38d..71b42c85b 100644 --- a/.github/workflows/nightly_ondemand_whl.yml +++ b/.github/workflows/nightly_ondemand_whl.yml @@ -244,7 +244,7 @@ jobs: # Print summary bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) - grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" tmp-*.txt |grep -E "failed|passed|timeout" -B 1 + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 if [ ${exit_label} -ne 0 ];then echo "There are ${exit_label} cases that need look into!!! Please check them" fi diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 48783856d..330364f58 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -196,7 +196,7 @@ jobs: # Print summary bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) - grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" tmp-*.txt |grep -E "failed|passed|timeout" -B 1 + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 if [ ${exit_label} -ne 0 ];then echo "There are ${exit_label} cases that need look into!!! Please check them" fi From 7809ac3a11fae1d5aa7c5834daecede74cd27106 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Thu, 20 Feb 2025 15:06:43 +0800 Subject: [PATCH 9/9] Update exit check --- .github/workflows/nightly_ondemand.yml | 5 +++-- .github/workflows/nightly_ondemand_rolling.yml | 5 +++-- .github/workflows/nightly_ondemand_whl.yml | 5 +++-- .github/workflows/pull.yml | 5 +++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml index 5866adb73..fa9daee67 100644 --- a/.github/workflows/nightly_ondemand.yml +++ b/.github/workflows/nightly_ondemand.yml @@ -286,13 +286,14 @@ jobs: find . -type f -name "*.tgz" -mtime +3 -delete # delete files older than 3 days tar zcf xpu-inductor-${GITHUB_RUN_ID}.tgz -C ${{ github.workspace }}/upload_files/ . # backup logs # Print summary + rm -rf /tmp/tmp-*.txt bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) - grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 if [ ${exit_label} -ne 0 ];then + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 echo "There are ${exit_label} cases that need look into!!! Please check them" + exit ${exit_label} fi - exit ${exit_label} - name: Upload Inductor XPU E2E Data if: ${{ ! cancelled() }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/nightly_ondemand_rolling.yml b/.github/workflows/nightly_ondemand_rolling.yml index 34f834e3c..2bf53e9eb 100644 --- a/.github/workflows/nightly_ondemand_rolling.yml +++ b/.github/workflows/nightly_ondemand_rolling.yml @@ -297,13 +297,14 @@ jobs: find . -type f -name "*.tgz" -mtime +3 -delete # delete files older than 3 days tar zcf xpu-inductor-${GITHUB_RUN_ID}.tgz -C ${{ github.workspace }}/upload_files/ . # backup logs # Print summary + rm -rf /tmp/tmp-*.txt bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) - grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 if [ ${exit_label} -ne 0 ];then + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 echo "There are ${exit_label} cases that need look into!!! Please check them" + exit ${exit_label} fi - exit ${exit_label} - name: Upload Inductor XPU E2E Data if: ${{ ! cancelled() }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/nightly_ondemand_whl.yml b/.github/workflows/nightly_ondemand_whl.yml index 71b42c85b..ec7f3835b 100644 --- a/.github/workflows/nightly_ondemand_whl.yml +++ b/.github/workflows/nightly_ondemand_whl.yml @@ -242,13 +242,14 @@ jobs: find . -type f -name "*.tgz" -mtime +3 -delete # delete files older than 3 days tar zcf xpu-inductor-${GITHUB_RUN_ID}.tgz -C ${{ github.workspace }}/upload_files/ . # backup logs # Print summary + rm -rf /tmp/tmp-*.txt bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) - grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 if [ ${exit_label} -ne 0 ];then + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 echo "There are ${exit_label} cases that need look into!!! Please check them" + exit ${exit_label} fi - exit ${exit_label} - name: Upload Inductor XPU E2E Data if: ${{ ! cancelled() }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 330364f58..fda8ce5c6 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -194,13 +194,14 @@ jobs: rm -rf ${{ github.workspace }}/upload_files cp -r ${{ github.workspace }}/../pytorch/inductor_log ${{ github.workspace }}/upload_files # Print summary + rm -rf /tmp/tmp-*.txt bash ${{ github.workspace }}/.github/scripts/e2e_summary.sh ${{ github.workspace }}/upload_files >> ${GITHUB_STEP_SUMMARY} exit_label=$(awk 'BEGIN{sum=0}{if($2>0){sum++}}END{print sum}' /tmp/tmp-result.txt) - grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 if [ ${exit_label} -ne 0 ];then + grep -E "(Real failed|to passed|Warning timeout).*: [1-9]|Summary for" /tmp/tmp-*.txt |grep -E "failed|passed|timeout" -B 1 echo "There are ${exit_label} cases that need look into!!! Please check them" + exit ${exit_label} fi - exit ${exit_label} - name: Upload Inductor XPU E2E Data if: ${{ ! cancelled() }} uses: actions/upload-artifact@v4