From b1778837d5541802e62d5033a51d4d4601a029e5 Mon Sep 17 00:00:00 2001 From: zhouchangda Date: Tue, 14 Jan 2025 15:25:33 +0000 Subject: [PATCH] update paddlex ci --- models/PaddleX/ci/black_list.txt | 51 +++ models/PaddleX/ci/ci_run.sh | 245 +++++++++--- models/PaddleX/ci/config.txt | 138 +++++-- models/PaddleX/ci/pipeline_config.txt | 20 + models/PaddleX/ci/pr_list.txt | 64 ++-- models/PaddleX/ci/pr_list_dcu.txt | 2 +- models/PaddleX/ci/pr_list_mlu.txt | 8 +- models/PaddleX/ci/pr_list_npu.txt | 18 +- models/PaddleX/ci/{checker.py => tools.py} | 416 +++++++++++++++------ 9 files changed, 717 insertions(+), 245 deletions(-) create mode 100644 models/PaddleX/ci/pipeline_config.txt rename models/PaddleX/ci/{checker.py => tools.py} (52%) diff --git a/models/PaddleX/ci/black_list.txt b/models/PaddleX/ci/black_list.txt index 3b4974229f..645151bcde 100644 --- a/models/PaddleX/ci/black_list.txt +++ b/models/PaddleX/ci/black_list.txt @@ -1,9 +1,60 @@ ################################### All ################################## ################################## Train ################################# +Train:PP-OCRv3_mobile_rec +Train:en_PP-OCRv3_mobile_rec +Train:korean_PP-OCRv3_mobile_rec +Train:japan_PP-OCRv3_mobile_rec +Train:chinese_cht_PP-OCRv3_mobile_rec +Train:te_PP-OCRv3_mobile_rec +Train:ka_PP-OCRv3_mobile_rec +Train:ta_PP-OCRv3_mobile_rec +Train:latin_PP-OCRv3_mobile_rec +Train:arabic_PP-OCRv3_mobile_rec +Train:cyrillic_PP-OCRv3_mobile_rec +Train:devanagari_PP-OCRv3_mobile_rec +Train:UniMERNet +Train:PP-FormulaNet-S +Train:PP-FormulaNet-L +Train:SLANeXt_wired ################################ Evaluate ################################ +Evaluate:PP-OCRv3_mobile_rec +Evaluate:en_PP-OCRv3_mobile_rec +Evaluate:korean_PP-OCRv3_mobile_rec +Evaluate:japan_PP-OCRv3_mobile_rec +Evaluate:chinese_cht_PP-OCRv3_mobile_rec +Evaluate:te_PP-OCRv3_mobile_rec +Evaluate:ka_PP-OCRv3_mobile_rec +Evaluate:ta_PP-OCRv3_mobile_rec +Evaluate:latin_PP-OCRv3_mobile_rec +Evaluate:arabic_PP-OCRv3_mobile_rec +Evaluate:cyrillic_PP-OCRv3_mobile_rec +Evaluate:devanagari_PP-OCRv3_mobile_rec +Evaluate:UniMERNet +Evaluate:PP-FormulaNet-S +Evaluate:PP-FormulaNet-L +Evaluate:SLANeXt_wired ################################# Predict ################################ +Predict:PP-OCRv3_mobile_rec +Predict:en_PP-OCRv3_mobile_rec +Predict:korean_PP-OCRv3_mobile_rec +Predict:japan_PP-OCRv3_mobile_rec +Predict:chinese_cht_PP-OCRv3_mobile_rec +Predict:te_PP-OCRv3_mobile_rec +Predict:ka_PP-OCRv3_mobile_rec +Predict:ta_PP-OCRv3_mobile_rec +Predict:latin_PP-OCRv3_mobile_rec +Predict:arabic_PP-OCRv3_mobile_rec +Predict:cyrillic_PP-OCRv3_mobile_rec +Predict:devanagari_PP-OCRv3_mobile_rec +Predict:UniMERNet +Predict:PP-FormulaNet-S +Predict:PP-FormulaNet-L Predict:TimesNet_ad +Predict:SLANeXt_wired ################################## Export ################################ +Export:UniMERNet +Export:PP-FormulaNet-S +Export:PP-FormulaNet-L ################################# Pipeline ############################### Pipeline:PP-ChatOCRv3-doc Pipeline:PP-ShiTuV2 diff --git a/models/PaddleX/ci/ci_run.sh b/models/PaddleX/ci/ci_run.sh index aea3eac930..e818bb1aaa 100644 --- a/models/PaddleX/ci/ci_run.sh +++ b/models/PaddleX/ci/ci_run.sh @@ -12,9 +12,13 @@ MODEL_LIST_FILE=$2 # WITHOUT_MD_NUM: PR中MD文件改动之外的改动文件数量，用于判断进行文档超链接检测后是否进行正常的CI，默认为空，设置示例：export WITHOUT_MD_NUM=10 if [[ $MODE == 'PaddleX' ]];then - set -x + # set -x failed_cmd_list="" +else + # set -x + set +e fi +successed_cmd_list="" #################################################### Functions ###################################################### @@ -37,7 +41,7 @@ function func_parser_dataset_url(){ function get_device_list(){ id_list=$DEVICE_ID - if [[ $suite_name == "PaddleTS" ]];then + if [[ $suite_name == "PaddleTS" || $mode == "predict" || $model_name == "YOWO" ]];then id_list=$FIRST_ID fi echo ${DEVICE_TYPE}:$id_list @@ -46,31 +50,54 @@ function get_device_list(){ # 运行命令并输出结果，PR级CI失败会重跑3次并异常退出，增量级和全量级会记录失败命令，最后打印失败的命令并异常退出 function run_command(){ command=$1 - module_name=$2 + module_name_=$2 time_stamp=$(date +"%Y-%m-%d %H:%M:%S") command="timeout 30m ${command}" printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}" - eval $command + if [[ -z $CI_DEBUG ]];then + sync + echo 1 > /proc/sys/vm/drop_caches + eval $command + else + echo $command + fi last_status=${PIPESTATUS[0]} n=1 + retry_time=3 # Try 2 times to run command if it fails - if [[ $MODE != 'PaddleX' ]];then - while [[ $last_status != 0 ]]; do - sleep 10 - n=`expr $n + 1` - printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}" + while [[ $last_status != 0 ]]; do + if [[ $last_status == 137 ]];then + echo "CI 因内存资源耗尽而中断，将于90秒后自动重试..." + retry_time=6 + sleep 90 sync echo 1 > /proc/sys/vm/drop_caches - eval $command - last_status=${PIPESTATUS[0]} - if [[ $n -eq 2 && $last_status != 0 ]]; then - echo "Retry 2 times failed with command: ${command}" + else + sleep 10 + fi + n=`expr $n + 1` + printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}" + eval $command + last_status=${PIPESTATUS[0]} + if [[ $n -ge $retry_time && $last_status != 0 ]]; then + if [[ $last_status == 137 ]];then + echo "CI 因内存资源耗尽而退出，如果在log中存在Kill字样，建议等待一段时间后重跑,如果连续重跑失败，请联系CI负责人排查问题。" + fi + if [[ $MODE != 'PaddleX' ]];then + echo "Retry $retry_time times failed with command: ${command}" exit 1 + else + break fi - done - elif [[ $last_status != 0 ]]; then - failed_cmd_list="$failed_cmd_list \n ${module_name} | command: ${command}" - echo "Run ${command} failed" + fi + done + if [[ $MODE == 'PaddleX' ]];then + if [[ $last_status != 0 ]]; then + failed_cmd_list="$failed_cmd_list\n${command}" + echo "CI_FAILED_CMD: ${command}" + else + successed_cmd_list="$successed_cmd_list\n${command}" + fi fi } @@ -80,12 +107,12 @@ function prepare_dataset(){ train_data_file="" return fi - download_dataset_cmd="${PYTHON_PATH} ${BASE_PATH}/checker.py --download_dataset --config_path ${check_dataset_yaml} --dataset_url ${dataset_url}" + download_dataset_cmd="${PYTHON_PATH} ${BASE_PATH}/tools.py --download_dataset --config_path ${check_dataset_yaml} --dataset_url ${dataset_url}" run_command ${download_dataset_cmd} ${module_name} model_output_path=${MODULE_OUTPUT_PATH}/${module_name}_dataset_check check_dataset_cmd="${PYTHON_PATH} main.py -c ${check_dataset_yaml} -o Global.mode=check_dataset -o Global.output=${model_output_path} " run_command ${check_dataset_cmd} ${module_name} - checker_cmd="${PYTHON_PATH} ${BASE_PATH}/checker.py --check --check_dataset_result --output ${model_output_path} --module_name ${module_name}" + checker_cmd="${PYTHON_PATH} ${BASE_PATH}/tools.py --check --check_dataset_result --output ${model_output_path} --module_name ${module_name}" run_command ${checker_cmd} ${module_name} dataset_dir=`cat $check_dataset_yaml | grep -m 1 dataset_dir | awk {'print$NF'}| sed 's/"//g'` if [[ ! -z $train_list_name ]]; then @@ -140,18 +167,36 @@ function run_models(){ fi mkdir -p $model_output_path IFS=$'|' + check_flag="" run_model_list=(${run_model}) + #### 有新推理接入，使用新推理 #### + if [[ $USE_NEW_INFERENCE == 1 ]];then + set_flag=`ls paddlex/inference/models_new | grep ${module_name}` + if [[ -z $set_flag ]];then + export PADDLE_PDX_NEW_PREDICTOR=0 + else + export PADDLE_PDX_NEW_PREDICTOR=1 + fi + fi for mode in ${run_model_list[@]};do black_model=`eval echo '$'"${mode}_black_list"|grep "^${model_name}$"` if [[ ! -z $black_model ]];then # 黑名单模型，不运行 + if [[ $mode == "train" ]];then + check_flag="False" + fi echo "$model_name is in ${mode}_black_list, so skip it." continue fi + device_info=$(get_device_list) # TEST_RANGE 为空时为普通全流程测试 if [[ -z $TEST_RANGE ]];then # 适配导出模型时，需要指定输出路径 - device_info=$(get_device_list) + if [[ $model_name == PP-YOLOE_plus-M || $model_name == PP-YOLOE_plus-S || $model_name == PP-YOLOE_plus-X ]];then + epochs_iters=5 + elif [[ $model_name == SOLOv2 ]];then + epochs_iters=10 + fi base_mode_cmd="${PYTHON_PATH} main.py -c ${config_path} -o Global.mode=${mode} -o Global.device=${device_info} -o Train.epochs_iters=${epochs_iters} -o Train.batch_size=${batch_size} -o Evaluate.weight_path=${evaluate_weight_path} -o Predict.model_dir=${inference_weight_path}" if [[ $mode == "export" ]];then model_export_output_path=${model_output_path}/export @@ -164,7 +209,7 @@ function run_models(){ run_command ${run_mode_cmd} ${module_name} # TEST_RANGE 为inference时，只测试官方模型预测 elif [[ $TEST_RANGE == "inference" && $mode == "predict" ]];then - offcial_model_predict_cmd="${PYTHON_PATH} main.py -c ${config_path} -o Global.mode=predict -o Predict.model_dir=None -o Global.output=${model_output_path}_offical_predict" + offcial_model_predict_cmd="${PYTHON_PATH} main.py -c ${config_path} -o Global.mode=predict -o Predict.model_dir=None -o Global.output=${model_output_path}_offical_predict -o Global.device=${device_info}" run_command ${offcial_model_predict_cmd} ${module_name} continue # TEST_RANGE 为其他非空值时，不做模型级别测试 @@ -172,7 +217,7 @@ function run_models(){ continue fi done - if [[ ! -z $black_model ]];then + if [[ ! -z $check_flag ]];then # 黑名单模型，不做检查 echo "$model_name is in ${mode}_black_list, so skip it." continue @@ -183,12 +228,16 @@ function run_models(){ # runing_train为空时为普通全过程测试 if [[ -z $TEST_RANGE ]];then check_options_list=(${check_options}) + rm_output="" for check_option in ${check_options_list[@]};do # 运行产出检查脚本 - checker_cmd="${PYTHON_PATH} ${BASE_PATH}/checker.py --check --$check_option --output ${model_output_path} --check_weights_items ${check_weights_items} --module_name ${module_name}" + checker_cmd="${PYTHON_PATH} ${BASE_PATH}/tools.py --check --$check_option --output ${model_output_path} --check_weights_items ${check_weights_items} --module_name ${module_name} 2>&1 | tee ${model_output_path}/checker_${check_option}.log" run_command ${checker_cmd} ${module_name} + if [[ $last_status != 0 ]];then + rm_output="${rm_output} false" + fi done - if [[ $last_status -eq 0 ]];then + if [[ -z $rm_output ]];then rm -rf ${model_output_path} fi fi @@ -222,15 +271,20 @@ PYTHON_PATH="python" BASE_PATH=$(cd "$(dirname $0)"; pwd) MODULE_OUTPUT_PATH=${BASE_PATH}/outputs CONFIG_FILE=${BASE_PATH}/config.txt -pip config set global.index-url https://mirrors.bfsu.edu.cn/pypi/web/simple -pip install beautifulsoup4==4.12.3 -pip install tqdm -pip install markdown +echo $CI_DEBUG +if [[ -z $CI_DEBUG ]];then + pip config set global.index-url https://mirrors.bfsu.edu.cn/pypi/web/simple + pip install beautifulsoup4==4.12.3 + pip install tqdm + pip install markdown + pip install prettytable + pip install colorlog +fi declare -A weight_dict declare -A model_dict #################################################### 代码风格检查 ###################################################### -if [[ DEVICE_TYPE == 'gpu' ]];then +if [[ $DEVICE_TYPE == 'gpu' ]];then pre-commit last_status=${PIPESTATUS[0]} if [[ $last_status != 0 ]]; then @@ -241,7 +295,7 @@ fi #################################################### 文档超链接检查 ###################################################### if [[ ! $MD_NUM -eq 0 && ! -z $MD_NUM ]];then - checker_url_cmd="${PYTHON_PATH} ${BASE_PATH}/checker.py --check_url -m internal" + checker_url_cmd="${PYTHON_PATH} ${BASE_PATH}/tools.py --check_url -m internal" eval $checker_url_cmd last_status=${PIPESTATUS[0]} if [[ $last_status != 0 ]]; then @@ -255,11 +309,22 @@ fi # 安装paddlex，完成环境准备 install_pdx_cmd="pip install -e ." -eval $install_pdx_cmd +if [[ -z $CI_DEBUG ]];then + eval $install_pdx_cmd + last_status=${PIPESTATUS[0]} + if [[ $last_status != 0 ]]; then + if [[ $last_status == 137 ]];then + echo "CI 因内存资源耗尽而退出，如果在log中存在Kill字样，建议等待一段时间后重跑,如果连续重跑失败，请联系CI负责人排查问题。" + else + echo "install paddlex failed, please fix it first." + fi + exit 1 + fi +fi if [[ -z $MEM_SIZE ]]; then - MEM_SIZE=16 + MEM_SIZE=32 fi if [[ -z $DEVICE_TYPE ]]; then @@ -285,15 +350,32 @@ else fi # # 只测试产线推理无需安装套件库 -if [[ $TEST_RANGE != "pipeline" ]];then +if [[ $TEST_RANGE != "pipeline" || -z $CI_DEBUG ]];then eval ${install_deps_cmd} + last_status=${PIPESTATUS[0]} + if [[ $last_status != 0 ]]; then + if [[ $last_status == 137 ]];then + echo "CI 因内存资源耗尽而退出，如果在log中存在Kill字样，建议等待一段时间后重跑,如果连续重跑失败，请联系CI负责人排查问题。" + else + echo "install suite repos failed, please fix it first." + fi + exit 1 + fi fi pip freeze > all_packages.txt +cmd="${PYTHON_PATH} ${BASE_PATH}/tools.py --check_env" +eval $cmd +last_status=${PIPESTATUS[0]} +if [[ $last_status != 0 ]]; then + echo "check env failed, please fix it first." + exit 1 +fi #################################################### 模型级测试 ###################################################### IFS=$' ' black_list_file=${BASE_PATH}/black_list.txt +set +e all_black_list=`cat ${black_list_file} | grep All: | awk -F : {'print$2'}` train_black_list=`cat ${black_list_file} | grep Train: | awk -F : {'print$2'}` train_black_list="$all_black_list @@ -308,7 +390,7 @@ export_black_list=`cat ${black_list_file} | grep Export: | awk -F : {'print$2'}` export_black_list="$all_black_list $export_black_list" pipeline_black_list=`cat ${black_list_file} | grep Pipeline: | awk -F : {'print$2'}` -echo "----------------------- Black list info ------------------------ +echo "======================= 黑名单信息 =========================== ##############train_black_list############### $train_black_list ##############evaluate_black_list############### @@ -319,7 +401,9 @@ $predict_black_list $export_black_list ##############pipeline_black_list############### $pipeline_black_list ------------------------------------------------------------------" +===============================================================" + +echo "============================ 开始模型级CI测试 =====================================" IFS='*' modules_info_list=($(cat ${CONFIG_FILE})) @@ -361,11 +445,11 @@ for modules_info in ${modules_info_list[@]}; do if [[ $MODE == "PaddleX" ]];then if [[ ! -z $MODEL_LIST_FILE ]];then new_model_info=`cat $MODEL_LIST_FILE` - new_model_module_names=`cat $MODEL_LIST_FILE | awk -F '/' {'print$3'} | sort -u` + new_model_module_names=`cat $MODEL_LIST_FILE | awk -F '/' {'print$4'} | sort -u` for new_module_info in ${new_model_module_names[@]};do module=`echo "${all_module_names[@]}" | grep $new_module_info` if [[ -z $module ]];then - echo "new module: $new_module_info is unsupported! Please contact with the developer or add new module info in ci_info.txt!" + echo "检测到未支持的module类型: $new_module_info ，请准备对应模块的测试数据集（为保障测试时间，测试数据集越小越好），并联系CI添加该模块的相关配置后重跑CI！" exit 1 fi if [[ $new_module_info == $module_name ]];then @@ -375,15 +459,15 @@ for modules_info in ${modules_info_list[@]}; do fi done else - module_info=`ls paddlex/configs/${module_name} | xargs -n1 -I {} echo config_path:paddlex/configs/${module_name}/{}` + module_info=`ls paddlex/configs/modules/${module_name} | xargs -n1 -I {} echo config_path:paddlex/configs/modules/${module_name}/{}` prepare_and_run "${module_info}" fi continue elif [[ $MODE == $suite_name ]];then - module_info=`cat ${BASE_PATH}/pr_list.txt | grep -v "^#" | grep paddlex/configs/${module_name}` + module_info=`cat ${BASE_PATH}/pr_list.txt | grep -v "^#" | grep paddlex/configs/modules/${module_name}` prepare_and_run "${module_info}" elif [[ -z $MODE ]];then - module_info=`cat ${BASE_PATH}/pr_list.txt | grep -v "^#" | grep paddlex/configs/${module_name}` + module_info=`cat ${BASE_PATH}/pr_list.txt | grep -v "^#" | grep paddlex/configs/modules/${module_name}` prepare_and_run "${module_info}" else continue @@ -395,9 +479,30 @@ done if [[ $PIPE_TYPE != 'gpu' ]];then exit 0 fi + +if [[ $LOCAL_RUN_WO_PIPLINE == 1 ]];then + if [[ $MODE == 'PaddleX' && ! -z $failed_cmd_list ]];then + echo "以下为失败的命令列表：" + echo -e "$failed_cmd_list" + echo "可在全量日志中检索“CI_FAILED_CMD”关键字，快速定位到运行失败命令位置，相关日志在关键字上方。如果有“Kill”相关字样，可以忽略问题重跑；如果是产出检查失败，建议继续向上排查，CI顺序为【数据->训练->评估->预测->导出】" + exit 1 + fi + exit 0 +fi + +if [[ ! -z $USE_NEW_INFERENCE ]];then + export USE_NEW_INFERENCE="" +fi + +echo "============================ 开始产线级CI测试 =====================================" + #################################################### 产线级测试 ###################################################### IFS=$'\n' -PIPELINE_YAML_LIST=`ls paddlex/pipelines | grep .yaml` +if [[ ! -z $USE_NEW_INFERENCE ]];then + PIPELINE_YAML_LIST=`ls paddlex/configs/pipelines | grep .yaml` +else + PIPELINE_YAML_LIST=`ls paddlex/pipelines | grep .yaml` +fi function check_pipeline() { pipeline=$1 @@ -409,18 +514,52 @@ function check_pipeline() { rm -rf $output_path mkdir -p $output_path cd $output_path - cmd="timeout 30m paddlex --pipeline ${pipeline} --input ${img} --device ${DEVICE_TYPE}:${FIRST_ID}" + cmd="timeout 30m paddlex --pipeline ${pipeline} --input ${img} --device ${DEVICE_TYPE}:${FIRST_ID} 2>&1 | tee ${BASE_PATH}/outputs/${pipeline}.log" echo $cmd - eval $cmd + if [[ -z $CI_DEBUG ]];then + eval $cmd + fi last_status=${PIPESTATUS[0]} - if [[ $last_status != 0 ]];then - exit 1 + n=1 + retry_time=3 + # Try 2 times to run command if it fails + if [[ $MODE != 'PaddleX' ]];then + while [[ $last_status != 0 ]]; do + if [[ $last_status == 137 ]];then + echo "CI 因内存资源耗尽而中断，将于90秒后自动重试..." + retry_time=6 + sleep 90 + sync + echo 1 > /proc/sys/vm/drop_caches + fi + sleep 10 + n=`expr $n + 1` + printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${cmd}" + eval $cmd + last_status=${PIPESTATUS[0]} + if [[ $n -eq $retry_time && $last_status != 0 ]]; then + if [[ $last_status == 137 ]];then + echo "CI 因内存资源耗尽而退出，如果在log中存在Kill字样，建议等待一段时间后重跑,如果连续重跑失败，请联系CI负责人排查问题。" + fi + echo "Retry 2 times failed with command: ${cmd}" + exit 1 + fi + done + elif [[ $last_status != 0 ]]; then + failed_cmd_list="$failed_cmd_list\n${cmd}" + echo "Run ${cmd} failed" + else + successed_cmd_list="$successed_cmd_list\n${cmd}" fi cd - } for pipeline_yaml in ${PIPELINE_YAML_LIST[@]};do - pipeline_name=`cat paddlex/pipelines/${pipeline_yaml} | grep pipeline_name | awk {'print$2'}` + if [[ ! -z $USE_NEW_INFERENCE ]];then + pipeline_name=`cat paddlex/configs/pipelines/${pipeline_yaml} | grep pipeline_name | awk {'print$2'} | head -n 1` + else + pipeline_name=`cat paddlex/pipelines/${pipeline_yaml} | grep pipeline_name | awk {'print$2'}` + fi IFS=' ' black_pipeline=`echo ${pipeline_black_list}|grep "^${pipeline_name}$"` IFS=$'\n' @@ -429,17 +568,21 @@ for pipeline_yaml in ${PIPELINE_YAML_LIST[@]};do echo "$pipeline_name is in pipeline_black_list, so skip it." continue fi - input=`cat paddlex/pipelines/${pipeline_yaml} | grep input | awk {'print$2'}` + input=`cat ci/pipeline_config.txt | grep "${pipeline_name}: " | awk -F ": " {'print$2'}` check_pipeline $pipeline_name "" "" $input done -# 全量CI检查全部URL if [[ $MODE == 'PaddleX' && -z $MODEL_LIST_FILE ]];then - checker_url_cmd="${PYTHON_PATH} ${BASE_PATH}/checker.py --check_url -m all" - eval $checker_url_cmd + echo -e "${successed_cmd_list}" > ${BASE_PATH}/outputs/success_cmd_list.txt + echo -e "${failed_cmd_list}" > ${BASE_PATH}/outputs/failed_cmd_list.txt + save_result_cmd="${PYTHON_PATH} ${BASE_PATH}/tools.py --save_result --successed_cmd ${BASE_PATH}/outputs/success_cmd_list.txt --failed_cmd ${BASE_PATH}/outputs/failed_cmd_list.txt" + echo $save_result_cmd + eval $save_result_cmd fi if [[ $MODE == 'PaddleX' && ! -z $failed_cmd_list ]];then - echo $failed_cmd_list + echo "以下为失败的命令列表：" + echo -e "$failed_cmd_list" + echo "可在全量日志中检索“CI_FAILED_CMD”关键字，快速定位到运行失败命令位置，相关日志在关键字上方。如果有“Kill”相关字样，可以忽略问题重跑；如果是产出检查失败，建议继续向上排查，CI顺序为【数据->训练->评估->预测->导出】" exit 1 fi diff --git a/models/PaddleX/ci/config.txt b/models/PaddleX/ci/config.txt index a6b3414bbb..6eee3333d1 100644 --- a/models/PaddleX/ci/config.txt +++ b/models/PaddleX/ci/config.txt @@ -1,6 +1,6 @@ suite_name:PaddleOCR module_name:text_recognition -check_dataset_yaml:paddlex/configs/text_recognition/PP-OCRv4_mobile_rec.yaml +check_dataset_yaml:paddlex/configs/modules/text_recognition/PP-OCRv4_mobile_rec.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/ocr_rec_dataset_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -12,7 +12,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleOCR module_name:text_detection -check_dataset_yaml:paddlex/configs/text_detection/PP-OCRv4_mobile_det.yaml +check_dataset_yaml:paddlex/configs/modules/text_detection/PP-OCRv4_mobile_det.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/ocr_det_dataset_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -24,7 +24,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleOCR module_name:formula_recognition -check_dataset_yaml:paddlex/configs/formula_recognition/LaTeX_OCR_rec.yaml +check_dataset_yaml:paddlex/configs/modules/formula_recognition/LaTeX_OCR_rec.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/ocr_rec_latexocr_dataset_example.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -35,8 +35,8 @@ inference_weight_dir:best_accuracy/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleOCR -module_name:table_recognition -check_dataset_yaml:paddlex/configs/table_recognition/SLANet.yaml +module_name:table_structure_recognition +check_dataset_yaml:paddlex/configs/modules/table_structure_recognition/SLANet.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/table_rec_dataset_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -48,7 +48,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleOCR module_name:seal_text_detection -check_dataset_yaml:paddlex/configs/seal_text_detection/PP-OCRv4_mobile_seal_det.yaml +check_dataset_yaml:paddlex/configs/modules/seal_text_detection/PP-OCRv4_mobile_seal_det.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/ocr_curve_det_dataset_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -60,7 +60,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection module_name:object_detection -check_dataset_yaml:paddlex/configs/object_detection/PicoDet-S.yaml +check_dataset_yaml:paddlex/configs/modules/object_detection/PicoDet-S.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/det_coco_examples.tar train_list_name: run_model:train|evaluate|predict|export @@ -72,7 +72,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection module_name:rotated_object_detection -check_dataset_yaml:paddlex/configs/rotated_object_detection/PP-YOLOE-R_L.yaml +check_dataset_yaml:paddlex/configs/modules/rotated_object_detection/PP-YOLOE-R_L.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/rdet_dota_examples.tar train_list_name: run_model:train|evaluate|predict|export @@ -84,7 +84,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection module_name:face_detection -check_dataset_yaml:paddlex/configs/face_detection/PicoDet_LCNet_x2_5_face.yaml +check_dataset_yaml:paddlex/configs/modules/face_detection/PicoDet_LCNet_x2_5_face.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/widerface_coco_examples.tar train_list_name: run_model:train|evaluate|predict|export @@ -96,7 +96,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection module_name:human_detection -check_dataset_yaml:paddlex/configs/human_detection/PP-YOLOE-L_human.yaml +check_dataset_yaml:paddlex/configs/modules/human_detection/PP-YOLOE-L_human.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/widerperson_coco_examples.tar train_list_name: run_model:train|evaluate|predict|export @@ -108,7 +108,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection module_name:mainbody_detection -check_dataset_yaml:paddlex/configs/mainbody_detection/PP-ShiTuV2_det.yaml +check_dataset_yaml:paddlex/configs/modules/mainbody_detection/PP-ShiTuV2_det.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/mainbody_det_examples.tar train_list_name: run_model:train|evaluate|predict|export @@ -120,7 +120,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection module_name:small_object_detection -check_dataset_yaml:paddlex/configs/small_object_detection/PP-YOLOE_plus_SOD-S.yaml +check_dataset_yaml:paddlex/configs/modules/small_object_detection/PP-YOLOE_plus_SOD-S.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/small_det_examples.tar train_list_name: run_model:train|evaluate|predict|export @@ -132,7 +132,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection module_name:vehicle_detection -check_dataset_yaml:paddlex/configs/vehicle_detection/PP-YOLOE-L_vehicle.yaml +check_dataset_yaml:paddlex/configs/modules/vehicle_detection/PP-YOLOE-L_vehicle.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/vehicle_coco_examples.tar train_list_name: run_model:train|evaluate|predict|export @@ -143,8 +143,20 @@ inference_weight_dir:best_model/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection +module_name:keypoint_detection +check_dataset_yaml:paddlex/configs/modules/keypoint_detection/PP-TinyPose_128x96.yaml +dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/keypoint_coco_examples.tar +train_list_name: +run_model:train|evaluate|predict|export +check_options:check_train_result_json|check_eval_result_json +check_weights_items:2,pdparams,pdema,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info +evaluate_weight_path:best_model/best_model.pdparams +inference_weight_dir:best_model/inference +epochs_iters:2 +********************************************************************************************************************* +suite_name:PaddleDetection module_name:instance_segmentation -check_dataset_yaml:paddlex/configs/instance_segmentation/Mask-RT-DETR-H.yaml +check_dataset_yaml:paddlex/configs/modules/instance_segmentation/Mask-RT-DETR-H.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/instance_seg_coco_examples.tar train_list_name: run_model:train|evaluate|predict|export @@ -156,7 +168,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection module_name:layout_detection -check_dataset_yaml:paddlex/configs/layout_detection/PicoDet_layout_1x.yaml +check_dataset_yaml:paddlex/configs/modules/layout_detection/PicoDet_layout_1x.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/det_layout_examples.tar train_list_name: run_model:train|evaluate|predict|export @@ -166,9 +178,21 @@ evaluate_weight_path:best_model/best_model.pdparams inference_weight_dir:best_model/inference epochs_iters:2 ********************************************************************************************************************* +suite_name:PaddleDetection +module_name:table_cells_detection +check_dataset_yaml:paddlex/configs/modules/table_cells_detection/RT-DETR-L_wired_table_cell_det.yaml +dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/cells_det_coco_examples.tar +train_list_name: +run_model:train|evaluate|predict|export +check_options:check_train_result_json|check_eval_result_json +check_weights_items:2,pdparams,pdema,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info +evaluate_weight_path:best_model/best_model.pdparams +inference_weight_dir:best_model/inference +epochs_iters:2 +********************************************************************************************************************* suite_name:PaddleClas module_name:face_feature -check_dataset_yaml:paddlex/configs/face_feature/MobileFaceNet.yaml +check_dataset_yaml:paddlex/configs/modules/face_feature/MobileFaceNet.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/face_rec_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -180,7 +204,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas module_name:image_classification -check_dataset_yaml:paddlex/configs/image_classification/ResNet50.yaml +check_dataset_yaml:paddlex/configs/modules/image_classification/ResNet50.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/cls_flowers_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -192,7 +216,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas module_name:doc_text_orientation -check_dataset_yaml:paddlex/configs/doc_text_orientation/PP-LCNet_x1_0_doc_ori.yaml +check_dataset_yaml:paddlex/configs/modules/doc_text_orientation/PP-LCNet_x1_0_doc_ori.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/text_image_orientation.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -204,7 +228,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas module_name:image_multilabel_classification -check_dataset_yaml:paddlex/configs/image_multilabel_classification/ResNet50_ML.yaml +check_dataset_yaml:paddlex/configs/modules/image_multilabel_classification/ResNet50_ML.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/mlcls_nus_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -216,7 +240,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas module_name:pedestrian_attribute_recognition -check_dataset_yaml:paddlex/configs/pedestrian_attribute_recognition/PP-LCNet_x1_0_pedestrian_attribute.yaml +check_dataset_yaml:paddlex/configs/modules/pedestrian_attribute_recognition/PP-LCNet_x1_0_pedestrian_attribute.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/pedestrian_attribute_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -228,7 +252,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas module_name:vehicle_attribute_recognition -check_dataset_yaml:paddlex/configs/vehicle_attribute_recognition/PP-LCNet_x1_0_vehicle_attribute.yaml +check_dataset_yaml:paddlex/configs/modules/vehicle_attribute_recognition/PP-LCNet_x1_0_vehicle_attribute.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/vehicle_attribute_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -240,7 +264,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas module_name:textline_orientation -check_dataset_yaml:paddlex/configs/textline_orientation/PP-LCNet_x0_25_textline_ori.yaml +check_dataset_yaml:paddlex/configs/modules/textline_orientation/PP-LCNet_x0_25_textline_ori.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/textline_orientation_example_data.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -252,7 +276,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas module_name:image_feature -check_dataset_yaml:paddlex/configs/image_feature/PP-ShiTuV2_rec.yaml +check_dataset_yaml:paddlex/configs/modules/image_feature/PP-ShiTuV2_rec.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/Inshop_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -263,8 +287,20 @@ inference_weight_dir:best_model/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas +module_name:table_classification +check_dataset_yaml:paddlex/configs/modules/table_classification/PP-LCNet_x1_0_table_cls.yaml +dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/table_cls_examples.tar +train_list_name:train.txt +run_model:train|evaluate|predict|export +check_options:check_train_result_json|check_eval_result_json +check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info +evaluate_weight_path:best_model/best_model.pdparams +inference_weight_dir:best_model/inference +epochs_iters:2 +********************************************************************************************************************* +suite_name:PaddleClas module_name:image_unwarping -check_dataset_yaml:paddlex/configs/image_unwarping/UVDoc.yaml +check_dataset_yaml:paddlex/configs/modules/image_unwarping/UVDoc.yaml dataset_url:null train_list_name:null run_model:predict @@ -276,7 +312,7 @@ epochs_iters:null ********************************************************************************************************************* suite_name:PaddleSeg module_name:semantic_segmentation -check_dataset_yaml:paddlex/configs/semantic_segmentation/OCRNet_HRNet-W48.yaml +check_dataset_yaml:paddlex/configs/modules/semantic_segmentation/OCRNet_HRNet-W48.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/seg_optic_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -288,7 +324,7 @@ epochs_iters:20 ********************************************************************************************************************* suite_name:PaddleSeg module_name:image_anomaly_detection -check_dataset_yaml:paddlex/configs/image_anomaly_detection/STFPM.yaml +check_dataset_yaml:paddlex/configs/modules/image_anomaly_detection/STFPM.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/mvtec_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -300,10 +336,10 @@ epochs_iters:20 ********************************************************************************************************************* suite_name:PaddleTS module_name:ts_forecast -check_dataset_yaml:paddlex/configs/ts_forecast/DLinear.yaml +check_dataset_yaml:paddlex/configs/modules/ts_forecast/DLinear.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/ts_dataset_examples.tar train_list_name:train.csv -run_model:train|evaluate|predict +run_model:train|evaluate|predict|export check_options:check_train_result_json|check_eval_result_json check_weights_items:score,pdparams evaluate_weight_path:best_model/model.pdparams @@ -312,10 +348,10 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleTS module_name:ts_anomaly_detection -check_dataset_yaml:paddlex/configs/ts_anomaly_detection/TimesNet_ad.yaml +check_dataset_yaml:paddlex/configs/modules/ts_anomaly_detection/TimesNet_ad.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/ts_anomaly_examples.tar train_list_name:train.csv -run_model:train|evaluate|predict +run_model:train|evaluate|predict|export check_options:check_train_result_json|check_eval_result_json check_weights_items:score,pdparams evaluate_weight_path:best_model/model.pdparams @@ -324,10 +360,10 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleTS module_name:ts_classification -check_dataset_yaml:paddlex/configs/ts_classification/TimesNet_cls.yaml +check_dataset_yaml:paddlex/configs/modules/ts_classification/TimesNet_cls.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/ts_classify_examples.tar train_list_name: -run_model:train|evaluate|predict +run_model:train|evaluate|predict|export check_options:check_train_result_json|check_eval_result_json check_weights_items:score,pdparams evaluate_weight_path:best_model/model.pdparams @@ -336,7 +372,7 @@ epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleVideo module_name:video_classification -check_dataset_yaml:paddlex/configs/video_classification/PPTSM_ResNet50_k400_8frames_uniform.yaml +check_dataset_yaml:paddlex/configs/modules/video_classification/PP-TSM-R50_8frames_uniform.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/k400_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -345,3 +381,39 @@ check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info evaluate_weight_path:best_model/best_model.pdparams inference_weight_dir:best_model/inference epochs_iters:2 +********************************************************************************************************************* +suite_name:Paddle3D +module_name:bev_fusion_3D +check_dataset_yaml:paddlex/configs/modules/bev_fusion_3D/bevf_pp_2x8_1x_nusc.yaml +dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/nuscenes_demo.tar +train_list_name:train.txt +run_model:train|evaluate|predict|export +check_options:check_train_result_json|check_eval_result_json +check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info +evaluate_weight_path:best_model/best_model.pdparams +inference_weight_dir:best_model/inference +epochs_iters:2 +********************************************************************************************************************* +suite_name:PaddleSpeech +module_name:multilingual_speech_recognition +check_dataset_yaml:paddlex/configs/modules/multilingual_speech_recognition/whisper_base.yaml +dataset_url:null +train_list_name:null +run_model:predict +check_options:null +check_weights_items:null +evaluate_weight_path:null +inference_weight_dir:null +epochs_iters:null +********************************************************************************************************************* +suite_name:PaddleVideo +module_name:video_detection +check_dataset_yaml:paddlex/configs/modules/video_detection/YOWO.yaml +dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/video_det_examples.tar +train_list_name:train.txt +run_model:train|evaluate|predict|export +check_options:check_train_result_json|check_eval_result_json +check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info +evaluate_weight_path:best_model/best_model.pdparams +inference_weight_dir:best_model/inference +epochs_iters:2 diff --git a/models/PaddleX/ci/pipeline_config.txt b/models/PaddleX/ci/pipeline_config.txt new file mode 100644 index 0000000000..023833f1d3 --- /dev/null +++ b/models/PaddleX/ci/pipeline_config.txt @@ -0,0 +1,20 @@ +anomaly_detection: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/uad_grid.png +face_recognition: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/friends1.jpg +formula_recognition: https://paddle-model-ecology.bj.bcebos.com/paddlex/demo_image/general_formula_recognition.png +image_classification: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/general_image_classification_001.jpg +instance_segmentation: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/general_instance_segmentation_004.png +layout_parsing: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/demo_paper.png +multi_label_image_classification: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/general_image_classification_001.jpg +object_detection: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/general_object_detection_002.png +OCR: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/general_ocr_001.png +pedestrian_attribute_recognition: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/pedestrian_attribute_002.jpg +PP-ChatOCRv3-doc: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/contract.pdf +seal_recognition: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/seal_text_det.png +semantic_segmentation: https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/application/semantic_segmentation/makassaridn-road_demo.png +small_object_detection: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/small_object_detection.jpg +table_recognition: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/table_recognition.jpg +ts_ad: https://paddle-model-ecology.bj.bcebos.com/paddlex/ts/demo_ts/ts_ad.csv +ts_cls: https://paddle-model-ecology.bj.bcebos.com/paddlex/ts/demo_ts/ts_cls.csv +ts_fc: https://paddle-model-ecology.bj.bcebos.com/paddlex/ts/demo_ts/ts_fc.csv +vehicle_attribute_recognition: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/vehicle_attribute_002.jpg +doc_preprocessor: https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/img_rot180_demo.jpg diff --git a/models/PaddleX/ci/pr_list.txt b/models/PaddleX/ci/pr_list.txt index 0235897826..9adc0f5c14 100644 --- a/models/PaddleX/ci/pr_list.txt +++ b/models/PaddleX/ci/pr_list.txt @@ -1,36 +1,38 @@ # For PaddleOCR -config_path:paddlex/configs/text_recognition/PP-OCRv4_mobile_rec.yaml -config_path:paddlex/configs/text_detection/PP-OCRv4_mobile_det.yaml -config_path:paddlex/configs/table_recognition/SLANet.yaml +config_path:paddlex/configs/modules/text_recognition/PP-OCRv4_mobile_rec.yaml +config_path:paddlex/configs/modules/text_detection/PP-OCRv4_mobile_det.yaml +config_path:paddlex/configs/modules/table_structure_recognition/SLANet.yaml # For PaddleDetection -config_path:paddlex/configs/object_detection/PicoDet-S.yaml -config_path:paddlex/configs/object_detection/PP-YOLOE_plus-S.yaml -config_path:paddlex/configs/object_detection/RT-DETR-L.yaml -config_path:paddlex/configs/instance_segmentation/Mask-RT-DETR-L.yaml +config_path:paddlex/configs/modules/object_detection/PicoDet-S.yaml +config_path:paddlex/configs/modules/object_detection/PP-YOLOE_plus-S.yaml +config_path:paddlex/configs/modules/object_detection/RT-DETR-L.yaml +config_path:paddlex/configs/modules/instance_segmentation/Mask-RT-DETR-L.yaml +config_path:paddlex/configs/modules/layout_detection/PicoDet_layout_1x.yaml # For PaddleClas -config_path:paddlex/configs/image_classification/ResNet18.yaml -config_path:paddlex/configs/image_classification/PP-LCNet_x0_25.yaml -config_path:paddlex/configs/image_classification/MobileNetV2_x0_25.yaml -config_path:paddlex/configs/image_classification/MobileNetV3_small_x0_35.yaml -config_path:paddlex/configs/image_classification/PP-HGNet_small.yaml -config_path:paddlex/configs/image_classification/PP-HGNetV2-B0.yaml -config_path:paddlex/configs/image_classification/ConvNeXt_tiny.yaml -config_path:paddlex/configs/image_classification/CLIP_vit_base_patch16_224.yaml -config_path:paddlex/configs/image_classification/SwinTransformer_tiny_patch4_window7_224.yaml +config_path:paddlex/configs/modules/image_classification/ResNet18.yaml +config_path:paddlex/configs/modules/image_classification/PP-LCNet_x0_25.yaml +config_path:paddlex/configs/modules/image_classification/PP-LCNetV2_small.yaml +config_path:paddlex/configs/modules/image_classification/MobileNetV2_x0_25.yaml +config_path:paddlex/configs/modules/image_classification/MobileNetV3_small_x0_35.yaml +config_path:paddlex/configs/modules/image_classification/PP-HGNet_small.yaml +config_path:paddlex/configs/modules/image_classification/PP-HGNetV2-B0.yaml +config_path:paddlex/configs/modules/image_classification/ConvNeXt_tiny.yaml +config_path:paddlex/configs/modules/image_classification/CLIP_vit_base_patch16_224.yaml +config_path:paddlex/configs/modules/image_classification/SwinTransformer_tiny_patch4_window7_224.yaml # For PaddleSeg -config_path:paddlex/configs/semantic_segmentation/PP-LiteSeg-T.yaml -config_path:paddlex/configs/semantic_segmentation/OCRNet_HRNet-W48.yaml -config_path:paddlex/configs/semantic_segmentation/Deeplabv3-R50.yaml -config_path:paddlex/configs/semantic_segmentation/Deeplabv3_Plus-R50.yaml +config_path:paddlex/configs/modules/semantic_segmentation/PP-LiteSeg-T.yaml +config_path:paddlex/configs/modules/semantic_segmentation/OCRNet_HRNet-W48.yaml +config_path:paddlex/configs/modules/semantic_segmentation/Deeplabv3-R50.yaml +config_path:paddlex/configs/modules/semantic_segmentation/Deeplabv3_Plus-R50.yaml # For PaddleTS -config_path:paddlex/configs/ts_forecast/DLinear.yaml -config_path:paddlex/configs/ts_forecast/PatchTST.yaml -config_path:paddlex/configs/ts_forecast/TiDE.yaml -config_path:paddlex/configs/ts_forecast/Nonstationary.yaml -config_path:paddlex/configs/ts_forecast/TimesNet.yaml -config_path:paddlex/configs/ts_anomaly_detection/DLinear_ad.yaml -config_path:paddlex/configs/ts_anomaly_detection/Nonstationary_ad.yaml -config_path:paddlex/configs/ts_anomaly_detection/TimesNet_ad.yaml -config_path:paddlex/configs/ts_anomaly_detection/AutoEncoder_ad.yaml -config_path:paddlex/configs/ts_anomaly_detection/PatchTST_ad.yaml -config_path:paddlex/configs/ts_classification/TimesNet_cls.yaml +config_path:paddlex/configs/modules/ts_forecast/DLinear.yaml +config_path:paddlex/configs/modules/ts_forecast/PatchTST.yaml +config_path:paddlex/configs/modules/ts_forecast/TiDE.yaml +config_path:paddlex/configs/modules/ts_forecast/Nonstationary.yaml +config_path:paddlex/configs/modules/ts_forecast/TimesNet.yaml +config_path:paddlex/configs/modules/ts_anomaly_detection/DLinear_ad.yaml +config_path:paddlex/configs/modules/ts_anomaly_detection/Nonstationary_ad.yaml +config_path:paddlex/configs/modules/ts_anomaly_detection/TimesNet_ad.yaml +config_path:paddlex/configs/modules/ts_anomaly_detection/AutoEncoder_ad.yaml +config_path:paddlex/configs/modules/ts_anomaly_detection/PatchTST_ad.yaml +config_path:paddlex/configs/modules/ts_classification/TimesNet_cls.yaml diff --git a/models/PaddleX/ci/pr_list_dcu.txt b/models/PaddleX/ci/pr_list_dcu.txt index 82b3e610d2..edaab27ce2 100644 --- a/models/PaddleX/ci/pr_list_dcu.txt +++ b/models/PaddleX/ci/pr_list_dcu.txt @@ -1,2 +1,2 @@ # For PaddleClas -config_path:paddlex/configs/image_classification/ResNet18.yaml +config_path:paddlex/configs/modules/image_classification/ResNet18.yaml diff --git a/models/PaddleX/ci/pr_list_mlu.txt b/models/PaddleX/ci/pr_list_mlu.txt index 17d51e0534..428e99b738 100644 --- a/models/PaddleX/ci/pr_list_mlu.txt +++ b/models/PaddleX/ci/pr_list_mlu.txt @@ -1,6 +1,6 @@ # For PaddleClas -config_path:paddlex/configs/image_classification/ResNet18.yaml -config_path:paddlex/configs/image_classification/PP-LCNet_x0_25.yaml -config_path:paddlex/configs/image_classification/MobileNetV3_small_x0_35.yaml +config_path:paddlex/configs/modules/image_classification/ResNet18.yaml +config_path:paddlex/configs/modules/image_classification/PP-LCNet_x0_25.yaml +config_path:paddlex/configs/modules/image_classification/MobileNetV3_small_x0_35.yaml # For PaddleTS -config_path:paddlex/configs/ts_forecast/DLinear.yaml +config_path:paddlex/configs/modules/ts_forecast/DLinear.yaml diff --git a/models/PaddleX/ci/pr_list_npu.txt b/models/PaddleX/ci/pr_list_npu.txt index dfd28f6a71..71c134be8c 100644 --- a/models/PaddleX/ci/pr_list_npu.txt +++ b/models/PaddleX/ci/pr_list_npu.txt @@ -1,13 +1,13 @@ # For PaddleDetection -config_path:paddlex/configs/object_detection/RT-DETR-L.yaml +config_path:paddlex/configs/modules/object_detection/RT-DETR-L.yaml # For PaddleClas -config_path:paddlex/configs/image_classification/ResNet18.yaml -config_path:paddlex/configs/image_classification/PP-LCNet_x0_25.yaml -config_path:paddlex/configs/image_classification/MobileNetV3_small_x0_35.yaml -config_path:paddlex/configs/image_classification/PP-HGNetV2-B0.yaml -config_path:paddlex/configs/image_classification/CLIP_vit_base_patch16_224.yaml -config_path:paddlex/configs/image_classification/SwinTransformer_tiny_patch4_window7_224.yaml +config_path:paddlex/configs/modules/image_classification/ResNet18.yaml +config_path:paddlex/configs/modules/image_classification/PP-LCNet_x0_25.yaml +config_path:paddlex/configs/modules/image_classification/MobileNetV3_small_x0_35.yaml +config_path:paddlex/configs/modules/image_classification/PP-HGNetV2-B0.yaml +config_path:paddlex/configs/modules/image_classification/CLIP_vit_base_patch16_224.yaml +config_path:paddlex/configs/modules/image_classification/SwinTransformer_tiny_patch4_window7_224.yaml # For PaddleSeg -config_path:paddlex/configs/semantic_segmentation/OCRNet_HRNet-W48.yaml +config_path:paddlex/configs/modules/semantic_segmentation/OCRNet_HRNet-W48.yaml # For PaddleTS -config_path:paddlex/configs/ts_forecast/DLinear.yaml +config_path:paddlex/configs/modules/ts_forecast/DLinear.yaml diff --git a/models/PaddleX/ci/checker.py b/models/PaddleX/ci/tools.py similarity index 52% rename from models/PaddleX/ci/checker.py rename to models/PaddleX/ci/tools.py index ae1e779fbe..467054816a 100644 --- a/models/PaddleX/ci/checker.py +++ b/models/PaddleX/ci/tools.py @@ -13,10 +13,13 @@ import json import time import shutil +import logging import tarfile import argparse from pathlib import Path import requests +import colorlog +from prettytable import PrettyTable from markdown import markdown from bs4 import BeautifulSoup from tqdm import tqdm @@ -28,19 +31,26 @@ def parse_args(): """Parse the arguments""" parser = argparse.ArgumentParser() + # For check urls in doc parser.add_argument("--check_url", action="store_true", default=False) + parser.add_argument("--check_env", action="store_true", default=False) parser.add_argument("-d", "--dir", default="./docs", type=str, help="The directory to search for Markdown files.") parser.add_argument( "-m", "--mode", default="all", choices=["all", "internal", "external"], help="The type of links to check." ) + # For save ci result for json + parser.add_argument("--save_result", action="store_true", default=False) + parser.add_argument("--successed_cmd", type=str, default="") + parser.add_argument("--failed_cmd", type=str, default="") + # For download dataset parser.add_argument("--download_dataset", action="store_true", default=False) parser.add_argument("--module_name", type=str, default=False) parser.add_argument("--config_path", type=str, default=False) parser.add_argument("--dataset_url", type=str, default=False) + # For check paddlex output parser.add_argument("--check", action="store_true", default=False) parser.add_argument("--output", type=str, default=False) parser.add_argument("--check_weights_items", type=str, default=False) - parser.add_argument("--check_train_result_json", action="store_true", default=False) parser.add_argument("--check_train_config_content", action="store_true", default=False) parser.add_argument("--check_dataset_result", action="store_true", default=False) @@ -95,35 +105,36 @@ class PostTrainingChecker: """Post training checker class""" def __init__(self, args): - self.check_results = [] - self.check_flag = [] + self.check_flag = True + + def update_fail_flag(self, msg): + """Update check flag""" + print(msg) + self.check_flag = False - def check_train_json_content( - self, output_dir, module_name, check_weights_items, train_result_json, check_train_json_message - ): + def check_train_json_content(self, output_dir, module_name, check_weights_items, train_result_json): """Check train result json content""" - pass_flag = True if not os.path.exists(train_result_json): - check_train_json_message.append(f"检查失败：{train_result_json} 不存在.") - pass_flag = False + msg = f"train_result.json文件不存在,检查路径为：{train_result_json}" + self.update_fail_flag(msg) try: with open(train_result_json, "r") as file: json_data = json.load(file) except json.JSONDecodeError: - check_train_json_message.append(f"打开 {train_result_json} 文件失败.") - pass_flag = False + msg = f"无法解析 {train_result_json} 文件的内容." + self.update_fail_flag(msg) + + print("*" * 20, "开始检查train_result.json,文件内容如下：", "*" * 20) + print(json_data) + print("*" * 60) if not json_data.get("done_flag", False): - check_train_json_message.append("检查失败：训练未完成") - pass_flag = False - err_type = json_data.get("err_type", None) - err_msg = json_data.get("err_msg", None) - if err_type and err_msg: - check_train_json_message.append(f"报错类型：{err_type}") - check_train_json_message.append(f"报错信息：{err_msg}") - else: - check_train_json_message.append("检查失败：未正确返回报错信息") + msg = "train_result.json文件中done_flag字段值为false,\ + 说明训练没有成功,如果dict中有'err_type'和'err_msg'字段,\ + 则可以通过查看err_type和err_msg来判断具体原因。\ + 如果不是这种情况，建议检查训练是否被异常终止，或代码退出时未正确写入done_flag" + self.update_fail_flag(msg) else: if "ts" in module_name: inspection_item = [ @@ -136,28 +147,48 @@ def check_train_json_content( if file_key == "score": score = last_data.get(file_key) if score == "": - check_train_json_message.append(f"检查失败：{file_key} 不存在") - pass_flag = False + msg = "train_result.json文件中score字段结果为空" + self.update_fail_flag(msg) else: - file_path = os.path.join(output_dir, last_data.get(file_key)) + try: + file_path = os.path.join(output_dir, last_data.get(file_key)) + except: + file_path = "" + self.update_fail_flag(msg) if last_data.get(file_key) == "" or not os.path.exists(file_path): - check_train_json_message.append(f"检查失败：在best中，{file_key} 对应的文件 {file_path} 不存在或为空") - pass_flag = False + msg = f"检查失败：在训练结果中，{file_key} 对应的文件 {last_data.get(file_key)} 不存在或为空,\ + 对于该模型CI强制检查的key包括：{inspection_item}" + self.update_fail_flag(msg) else: config_path = json_data.get("config") visualdl_log_path = json_data.get("visualdl_log") label_dict_path = json_data.get("label_dict") - if not os.path.exists(os.path.join(output_dir, config_path)): - check_train_json_message.append(f"检查失败：配置文件 {config_path} 不存在") - pass_flag = False + try: + file_path = os.path.join(output_dir, config_path) + except: + file_path = "" + if not os.path.exists(file_path): + msg = f"根据json中的config字段信息，未找到配置文件，请确认配置文件是否存在，配置文件路径为：{os.path.join(output_dir, config_path)}" + self.update_fail_flag(msg) if not ("text" in module_name or "table" in module_name or "formula" in module_name): - if not os.path.exists(os.path.join(output_dir, visualdl_log_path)): - check_train_json_message.append(f"检查失败：VisualDL日志文件 {visualdl_log_path} 不存在") - pass_flag = False - - if not os.path.exists(os.path.join(output_dir, label_dict_path)): - check_train_json_message.append(f"检查失败：标签映射文件 {label_dict_path} 不存在") - pass_flag = False + try: + file_path = os.path.join(output_dir, visualdl_log_path) + except: + file_path = "" + if not os.path.exists(file_path): + msg = f"根据json中的visualdl_log字段信息，\ + 未找到VisualDL日志文件，请确认VisualDL日志文件是否存在，\ + VisualDL日志文件路径为:{output_dir}中的{visualdl_log_path}" + self.update_fail_flag(msg) + try: + file_path = os.path.join(output_dir, label_dict_path) + except: + file_path = "" + if not os.path.exists(file_path): + msg = f"根据json中的label_dict字段信息，未找到标签映射文件，\ + 请确认标签映射文件是否存在，标签映射文件路径为:\ + {output_dir}中的{label_dict_path}" + self.update_fail_flag(msg) inspection_item = check_weights_items.split(",")[1:] last_k = check_weights_items.split(",")[0] @@ -166,72 +197,72 @@ def check_train_json_content( last_data = json_data["models"].get(last_key) for file_key in inspection_item: - file_path = os.path.join(output_dir, last_data.get(file_key)) + try: + file_path = os.path.join(output_dir, last_data.get(file_key)) + except: + file_path = "" if last_data.get(file_key) == "" or not os.path.exists(file_path): - check_train_json_message.append(f"检查失败：在 {last_key} 中，{file_key} 对应的文件 {file_path} 不存在或为空") - pass_flag = False + msg = f"检查失败：在第{i}轮的训练结果中，\ + {file_key} 对应的文件 {last_data.get(file_key)} 不存在或为空,\ + 对于该模型CI强制检查的key包括：{inspection_item}" + self.update_fail_flag(msg) best_key = "best" best_data = json_data["models"].get(best_key) if best_data.get("score") == "": - check_train_json_message.append(f"检查失败：{best_key} 中，score 不存在或为空") - pass_flag = False + msg = "train_result.json文件中score字段结果为空" + self.update_fail_flag(msg) for file_key in inspection_item: - file_path = os.path.join(output_dir, best_data.get(file_key)) + try: + file_path = os.path.join(output_dir, best_data.get(file_key)) + except: + file_path = "" if best_data.get(file_key) == "" or not os.path.exists(file_path): - check_train_json_message.append(f"检查失败：在 {best_key} 中，{file_key} 对应的文件 {file_path} 不存在或为空") - pass_flag = False - return pass_flag, check_train_json_message + msg = f"检查失败：在最佳权重结果中，{file_key} 对应的文件 {file_path} 不存在或为空,对于该模型CI强制检查的key包括：{inspection_item}" + self.update_fail_flag(msg) + return self.check_flag - def check_dataset_json_content(self, output_dir, module_name, dataset_result_json, check_dataset_json_message): + def check_dataset_json_content(self, output_dir, module_name, dataset_result_json): """Check dataset result json content""" - pass_flag = True if not os.path.exists(dataset_result_json): - check_dataset_json_message.append(f"{dataset_result_json} 不存在.") + msg = f"check_dataset_result.json文件不存在,检查路径为：{dataset_result_json}" + self.update_fail_flag(msg) try: with open(dataset_result_json, "r") as file: json_data = json.load(file) except json.JSONDecodeError: - check_dataset_json_message.append(f"检查失败：打开 {dataset_result_json} 文件失败.") - pass_flag = False + msg = f"无法解析 {dataset_result_json} 文件的内容." + self.update_fail_flag(msg) + + print("*" * 20, "开始检查check_dataset_result.json,文件内容如下：", "*" * 20) + print(json_data) + print("*" * 60) if not json_data.get("check_pass", False): - check_dataset_json_message.append("检查失败：数据校验未通过") - pass_flag = False - err_type = json_data.get("err_type", None) - err_msg = json_data.get("err_msg", None) - if err_type and err_msg: - check_dataset_json_message.append(f"报错类型：{err_type}") - check_dataset_json_message.append(f"报错信息：{err_msg}") - else: - check_dataset_json_message.append("检查失败：未正确返回报错信息") - # 检查config和visualdl_log字段对应的文件是否存在 - dataset_path = json_data.get("dataset_path") - # if not os.path.exists(os.path.join(output_dir, dataset_path)): - if not os.path.exists(dataset_path): - check_dataset_json_message.append(f"检查失败：数据集路径 {dataset_path} 不存在") - pass_flag = False + msg = "检查失败：数据校验未通过,如果dict中有'err_type'和'err_msg'字段,则可以通过查看err_type和err_msg来判断具体原因。如果不是这种情况，建议检查数据集是否符合规范" + self.update_fail_flag(msg) + if "ts" in module_name: show_type = json_data.get("show_type") if show_type not in ["csv"]: - check_dataset_json_message.append(f"检查失败：{show_type} 必须为'csv'") - pass_flag = False + msg = f"对于TS任务，show_type 必须为'csv'，但是检测到的是 {show_type}" + self.update_fail_flag(msg) for tag in ["train", "val", "test"]: samples_key = f"{tag}_table" samples_list = json_data["attributes"].get(samples_key) if tag == "test" and not samples_list: continue if len(samples_list) == 0: - check_dataset_json_message.append(f"检查失败：在 {samples_key} 中，值为空") - pass_flag = False + msg = f"检查失败：在csv表格中，{samples_key} 列为空" + self.update_fail_flag(msg) else: show_type = json_data.get("show_type") if show_type not in ["image", "txt", "csv", "video"]: - check_dataset_json_message.append(f"检查失败：{show_type} 必须为'image', 'txt', 'csv','video'其中一个") - pass_flag = False + msg = f"对于非TS任务，show_type 必须为'image', 'txt', 'csv','video'其中一个，但是检测到的是 {show_type}" + self.update_fail_flag(msg) - if module_name == "general_recognition": + if module_name in ["general_recognition", "image_feature"]: tag_list = ["train", "gallery", "query"] else: tag_list = ["train", "val"] @@ -239,57 +270,59 @@ def check_dataset_json_content(self, output_dir, module_name, dataset_result_jso for tag in tag_list: samples_key = f"{tag}_sample_paths" samples_path = json_data["attributes"].get(samples_key) + if samples_path is None: + msg = f"检查失败：未能获取到{samples_key}字段" + self.update_fail_flag(msg) + continue for sample_path in samples_path: sample_path = os.path.abspath(os.path.join(output_dir, sample_path)) if not samples_path or not os.path.exists(sample_path): - check_dataset_json_message.append(f"检查失败：在 {samples_key} 中，{sample_path} 对应的文件不存在或为空") - pass_flag = False + msg = f"检查失败：{samples_key}字段对应的文件不存在或为空，文件路径为：{sample_path}" + self.update_fail_flag(msg) if not ( "text" in module_name or "table" in module_name or "formula" in module_name - or module_name == "general_recognition" - or module_name == "face_feature" + or module_name in ["image_feature", "face_feature", "general_recognition"] ): try: num_class = int(json_data["attributes"].get("num_classes")) except ValueError: - check_dataset_json_message.append(f"检查失败：{num_class} 为空或不为整数") - pass_flag = False - if "table" not in module_name: + msg = f"检查失败：{num_class} 为空或不为整数" + self.update_fail_flag(msg) + if "table" not in module_name and module_name != "image_feature" and module_name != "face_feature": analyse_path = json_data["analysis"].get("histogram") if not analyse_path or not os.path.exists(os.path.join(output_dir, analyse_path)): - check_dataset_json_message.append(f"检查失败：{analyse_path} 数据分析文件不存在或为空") - pass_flag = False + msg = f"检查失败：{analyse_path} 数据分析文件不存在或为空" + self.update_fail_flag(msg) - return pass_flag, check_dataset_json_message + return self.check_flag - def check_eval_json_content(self, module_name, eval_result_json, check_eval_json_message): + def check_eval_json_content(self, module_name, eval_result_json): """Check eval result json content""" - pass_flag = True if not os.path.exists(eval_result_json): - check_eval_json_message.append(f"检查失败：{eval_result_json} 不存在.") - pass_flag = False + msg = "eval_result.json文件不存在" + self.update_fail_flag(msg) try: with open(eval_result_json, "r") as file: json_data = json.load(file) except json.JSONDecodeError: - check_eval_json_message.append(f"打开 {eval_result_json} 文件失败.") - pass_flag = False + msg = f"无法解析 {eval_result_json} 文件的内容." + self.update_fail_flag(msg) + + print("*" * 20, "开始检查eval_result.json,文件内容如下：", "*" * 20) + print(json_data) + print("*" * 60) if not json_data.get("done_flag", False): - check_eval_json_message.append("检查失败：评估未完成") - pass_flag = False - err_type = json_data.get("err_type", None) - err_msg = json_data.get("err_msg", None) - if err_type and err_msg: - check_eval_json_message.append(f"报错类型：{err_type}") - check_eval_json_message.append(f"报错信息：{err_msg}") - else: - check_eval_json_message.append("检查失败：未正确返回报错信息") + msg = "eval_result.json文件中done_flag字段值为false,\ + 如果dict中有'err_type'和'err_msg'字段,\ + 则可以通过查看err_type和err_msg来判断具体原因。\ + 如果不是这种情况，建议检查评估产出是否正确" + self.update_fail_flag(msg) - return pass_flag, check_eval_json_message + return self.check_flag def check_split_dataset(self, output_dir, args_dict, check_splitdata_message): """Check the split of a dataset""" @@ -357,26 +390,17 @@ def run_checks(self, args): """Run all checks on the specified arguments""" output_dir = args.output module_name = args.module_name + print("=" * 20, "开始执行产出结果检查", "=" * 20) if args.check_dataset_result: # 检查 check_result.json 内容 dataset_result_json = os.path.join(output_dir, "check_dataset_result.json") - check_dataset_json_message = [] - check_dataset_json_falg, check_dataset_json_message = self.check_dataset_json_content( - output_dir, module_name, dataset_result_json, check_dataset_json_message - ) - self.check_results = self.check_results + check_dataset_json_message - self.check_flag.append(check_dataset_json_falg) + self.check_dataset_json_content(output_dir, module_name, dataset_result_json) if args.check_train_result_json: # 检查 train_result.json 内容 train_result_json = os.path.join(output_dir, "train_result.json") check_weights_items = args.check_weights_items - check_train_json_message = [] - check_train_json_flag, check_train_json_message = self.check_train_json_content( - output_dir, module_name, check_weights_items, train_result_json, check_train_json_message - ) - self.check_results = self.check_results + check_train_json_message - self.check_flag.append(check_train_json_flag) + self.check_train_json_content(output_dir, module_name, check_weights_items, train_result_json) if args.check_split_dataset: # 检查数据划分是否正确 @@ -392,14 +416,13 @@ def run_checks(self, args): if args.check_eval_result_json: # 检查 eval_result.json 内容 eval_result_json = os.path.join(output_dir, "evaluate_result.json") - check_eval_json_message = [] - check_eval_json_flag, check_eval_json_message = self.check_eval_json_content( - module_name, eval_result_json, check_eval_json_message - ) - self.check_results = self.check_results + check_eval_json_message - self.check_flag.append(check_eval_json_flag) + self.check_eval_json_content(module_name, eval_result_json) - assert False not in self.check_flag, print("校验检查失败，请查看产出", " ".join(str(item) for item in self.check_results)) + if self.check_flag: + print("=" * 20, "产出结果检查通过", "=" * 20) + else: + print("=" * 20, "产出结果检查失败,请根据以上错误信息进行排查修复", "=" * 20) + exit(1) ################################### 检查文档超链接 ############################################### @@ -507,8 +530,165 @@ def check_documentation_url(args): pass +def get_option(cmd): + "get_option" + if "--check" in cmd: + cmd_list = cmd.split(" ") + check_option = cmd_list[5] + model_name = cmd_list[7].split("/")[-1] + if check_option == "--check_train_result_json": + option = f"{model_name}_check_train_result_json" + elif check_option == "--check_eval_result_json": + option = f"{model_name}_check_eval_result_json" + elif check_option == "--check_dataset_result": + option = f"{model_name}_check_dataset_result" + logfile = cmd_list[-1] + return option, logfile + elif "--pipeline" in cmd: + cmd_list = cmd.split(" ") + pipeline_name = cmd_list[4] + logfile = cmd_list[-1] + return pipeline_name, logfile + elif "main.py" in cmd: + cmd_list = cmd.split(" ") + model_name = cmd_list[5].split("/")[-1].split(".yaml")[0] + option = cmd_list[7].split("=")[-1] + if option == "train": + logfile = cmd_list[-1].split("=")[-1] + logfile += "train.log" + else: + logfile = None + option = f"{model_name}_{option}" + return option, logfile + else: + return None, None + + +def save_result_json(args): + "save_result_json" + with open(args.successed_cmd, "r") as f: + successed_cmd_list = f.readlines() + with open(args.failed_cmd, "r") as f: + failed_cmd_list = f.readlines() + result = {"successed": [], "failed": []} + successed_num = len(successed_cmd_list) - 1 + failed_num = len(failed_cmd_list) - 1 + result["successed_num"] = successed_num + result["failed_num"] = failed_num + for i, cmd in enumerate(successed_cmd_list[1:]): + check_option, logfile = get_option(cmd.strip()) + if check_option: + result["successed"].append(check_option) + for i, cmd in enumerate(failed_cmd_list[1:]): + check_option, logfile = get_option(cmd.strip()) + if check_option: + try: + with open(logfile, "r") as f: + log_info = f.readlines() + except: + log_info = "未成功提取log，请在全局log中自行查找" + result["failed"].append({check_option: log_info}) + with open("ci/results.json", "w") as f: + json.dump(result, f, ensure_ascii=False) + # print(args.failed_cmd) + + +def get_logger(level=logging.INFO): + """ + 获取带有颜色的logger + """ + # 创建logger对象 + logger = logging.getLogger() + logger.setLevel(level) + # 创建控制台日志处理器 + console_handler = logging.StreamHandler() + console_handler.setLevel(level) + # 定义颜色输出格式 + color_formatter = colorlog.ColoredFormatter( + "%(log_color)s%(message)s", + log_colors={ + "DEBUG": "cyan", + "INFO": "green", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "red,bg_white", + }, + ) + # 将颜色输出格式添加到控制台日志处理器 + console_handler.setFormatter(color_formatter) + # 移除默认的handler + for handler in logger.handlers: + logger.removeHandler(handler) + # 将控制台日志处理器添加到logger对象 + logger.addHandler(console_handler) + return logger + + +def format_commit(commit): + """ + 格式化commit信息 + """ + commit = "-".join(commit[i : i + 2] for i in range(0, len(commit), 2)) + return commit + + +def check_environment(): + """检查环境是否满足要求""" + table = PrettyTable(["Environment", "Value", "Description"]) + device_type = os.environ.get("DEVICE_TYPE", None) + device_ids = os.environ.get("DEVICE_ID", None) + msg = "=" * 20 + "以下为当前环境变量配置信息" + "=" * 20 + logger.info(msg) + if not device_type and not device_ids: + logger.warning("未设置设备类型和编号，使用配置文件中的默认值") + + table.add_row(["Device", os.environ.get("DEVICE_TYPE", "GPU"), "算力卡类型，例如GPU、NPU、XPU等，默认为GPU"]) + table.add_row(["Device ID", os.environ.get("DEVICE_ID", "0,1,2,3"), "使用的设备id列表，多个用逗号分隔"]) + table.add_row(["MEM_SIZE", os.environ.get("MEM_SIZE", "32"), "显存大小，单位GB"]) + table.add_row(["TEST_RANGE", os.environ.get("TEST_RANGE", ""), "测试范围，默认为空，设置示例：export TEST_RANGE='inference'"]) + table.add_row(["MD_NUM", os.environ.get("MD_NUM", ""), "PR中MD文件改动数量，用于判断是否需要进行文档超链接检测，默认为空，设置示例：export MD_NUM=10"]) + table.add_row( + ["WITHOUT_MD_NUM", os.environ.get("WITHOUT_MD_NUM", ""), "PR中除去MD文件改动数量，默认为空，设置示例：export WITHOUT_MD_NUM=10"] + ) + logger.info(table) + msg = "=" * 20 + "以下为Paddle相关版本信息" + "=" * 20 + logger.info(msg) + logger.info("注：为了能够显示commit信息，在每个数字中增加了一个“-”连字符，实际使用请删除连字符") + table = PrettyTable(["Repository", "Branch", "Commit"]) + try: + import paddle + + commit = paddle.__git_commit__ + table.add_row(["PaddlePaddle", paddle.__version__, format_commit(commit)]) + except ImportError: + logger.error("Paddle is not installed,please install it first.") + exit(1) + try: + branch = os.popen("git rev-parse --abbrev-ref HEAD").read().strip() + commit = os.popen("git rev-parse HEAD").read().strip() + table.add_row(["PaddleX", branch, format_commit(commit)]) + except ImportError: + logger.error("PaddleX is not installed,please install it first.") + exit(1) + repos = os.listdir("paddlex/repo_manager/repos") + for repo in repos: + if repo == ".gitkeep": + continue + try: + branch = os.popen(f"cd paddlex/repo_manager/repos/{repo} && git rev-parse --abbrev-ref HEAD").read().strip() + commit = os.popen(f"cd paddlex/repo_manager/repos/{repo} &&git rev-parse HEAD").read().strip() + table.add_row([repo, branch, format_commit(commit)]) + except: + pass + _logger = get_logger() + _logger.info(table) + msg = "=" * 20 + "环境检测完成" + "=" * 20 + logger.info(msg) + + if __name__ == "__main__": check_items, args = parse_args() + logger = get_logger() if args.check: checker = PostTrainingChecker(args) checker.run_checks(args) @@ -516,3 +696,7 @@ def check_documentation_url(args): download_dataset(args) elif args.check_url: check_documentation_url(args) + elif args.save_result: + save_result_json(args) + elif args.check_env: + check_environment()