feat: some funcs

matrixorigin · Dec 2, 2024 · 8043132 · 8043132
1 parent 2abc3dc
commit 8043132
Show file tree

Hide file tree

Showing 6 changed files with 97 additions and 54 deletions.
diff --git a/bin/datax.sh b/bin/datax.sh
@@ -6,6 +6,8 @@
 # datax
 
 datax_name="datax"
+datax_db_name=""
+datax_table_name=""
 
 function datax_precheck()
 {
@@ -38,6 +40,20 @@ function datax_precheck()
 
 }
 
+function datax_get_db_and_table()
+{
+    datax_db_name=`grep "jdbc:" "${DATAX_CONF_PATH}" | awk -F "/" '{print $4}' | awk -F "?" '{print $1}'  | sort  | uniq | head -n 1`
+    if echo "${datax_db_name}" | grep "$" >/dev/null 2>&1 ; then
+        datax_db_name=`eval echo ${datax_db_name}`
+    fi
+    datax_table_name=`grep '"table":' "${DATAX_CONF_PATH}"  | awk -F ":" '{print $2}' | sed "s#\[##g" | sed "s#\]##g" | sed "s#\"##g" | sed "s# ##g" | sort | uniq | head -n 1`
+    if echo "${datax_table_name}" | grep "$" >/dev/null 2>&1 ; then
+        datax_table_name=`eval echo ${datax_table_name}`
+    fi
+
+    add_log "D" "datax_db_name: ${datax_db_name}, datax_table_name: ${datax_table_name}"
+}
+
 function datax_report_write()
 {
     this_date=`date '+%Y-%m-%d'`
@@ -47,7 +63,7 @@ function datax_report_write()
     add_log "D" "log_file: ${log_file}"
     if [ ! -s ${DATAX_REPORT_FILE} ]; then
         touch "${DATAX_REPORT_FILE}"
-        echo "start_time,end_time,duration,avg_data_size,avg_data_rows,rows_total,rows_failed" >> "${DATAX_REPORT_FILE}"
+        echo "start_time,end_time,duration,avg_data_size,avg_data_rows,rows_total,rows_failed,db_name,table_name" >> "${DATAX_REPORT_FILE}"
     fi
 
     key_list=("任务启动时刻" "任务结束时刻" "任务总计耗时" "任务平均流量" "记录写入速度" "读出记录总数" "读写失败总数")
@@ -58,25 +74,48 @@ function datax_report_write()
         value=`grep "${key}" "${log_file}" | awk -F ": " '{print $2}' | sed "s/^ *//g"`
         add_log "D" "k_count: ${k_count}, key: ${key}, value: ${value}"
         let k_count=k_count+1
-        if [[ ${k_count} -le ${key_length} ]]; then
-            echo -n "${value}," >> "${DATAX_REPORT_FILE}"
-        else
-            echo "${value}" >> "${DATAX_REPORT_FILE}"
-        fi
+
+        echo -n "${value}," >> "${DATAX_REPORT_FILE}"
+        #if [[ ${k_count} -le ${key_length} ]]; then
+        #    echo -n "${value}," >> "${DATAX_REPORT_FILE}"
+        #else
+        #    echo "${value}" >> "${DATAX_REPORT_FILE}"
+        #fi
     done
+
+    echo "${datax_db_name},${datax_table_name}" >> "${DATAX_REPORT_FILE}"
+
 }
 
 function datax_get_conf()
 {
     add_log "I" "Get datax related confs"
-    get_conf |grep DATAX
+    get_conf | grep DATAX
 }
 
 function datax_run()
 {
 
-    if [ -d ${DATAX_CONF_PATH} ]; then
-        add_log "I" "DATAX_CONF_PATH ${DATAX_CONF_PATH} is a directory, listing files in it"
+    add_log "D" "DATAX_PARA_NAME_LIST: ${DATAX_PARA_NAME_LIST}"
+    if [[ "${DATAX_PARA_NAME_LIST}" != "" ]]; then
+        add_log "D" "DATAX_PARA_NAME_LIST: ${DATAX_PARA_NAME_LIST} is set, will add -p option when calling datax jobs"
+        para_count=1
+        p_option=""
+        for para_name in $(echo "${DATAX_PARA_NAME_LIST}" | sed "s/,/ /g"); do
+            para_value=`eval echo '$'${para_name}`
+            add_log "D" "para_count: ${para_count}, para_name: ${para_name}, para_value: ${para_value}"
+            if [[ "${p_option}" == "" ]]; then
+                p_option="-D${para_name}=${para_value}"
+            else
+                p_option="${p_option} -D${para_name}=${para_value}"
+            fi
+            let para_count=para_count+1
+        done
+        add_log "D" "p_option: ${p_option}"
+    fi
+
+    if [[ -d ${DATAX_CONF_PATH} ]] || [[ -s ${DATAX_CONF_PATH} ]]; then
+        add_log "I" "DATAX_CONF_PATH ${DATAX_CONF_PATH} is a directory or a file, listing files in it"
         add_log "I" "---------------------------------"
         ls -lth ${DATAX_CONF_PATH}
         add_log "I" "---------------------------------"
@@ -87,14 +126,35 @@ function datax_run()
         i=1
         failed="0"
         for conf_file in `ls ${DATAX_CONF_PATH}`; do
-            add_log "I" "File No.: ${i}, file_path: ${DATAX_CONF_PATH}/${conf_file}"
-            if ! python3 ${DATAX_TOOL_PATH}/bin/datax.py ${DATAX_CONF_PATH}/${conf_file}; then
-                add_log "E" "Result: failed"
-                let failed=failed+1
+            datax_get_db_and_table
+            basename_conf_file=`basename ${conf_file}`
+            conf_dir="${DATAX_CONF_PATH}"
+            if  [[ -s ${DATAX_CONF_PATH} ]]; then
+                conf_dir=`dirname "${DATAX_CONF_PATH}"`
+            fi
+
+            add_log "I" "File No.: ${i}, file_path: ${conf_dir}/${basename_conf_file}"
+            if [[ "${p_option}" != "" ]]; then
+                add_log "D" "cmd: python3 ${DATAX_TOOL_PATH}/bin/datax.py -p ${p_option} ${conf_dir}/${basename_conf_file}"
+                if ! python3 ${DATAX_TOOL_PATH}/bin/datax.py -p "${p_option}" ${conf_dir}/${basename_conf_file}; then
+                    add_log "E" "Result: failed"
+                    let failed=failed+1
+                else
+                    add_log "I" "Result: success"
+                    datax_report_write
+                fi
             else
-                add_log "I" "Result: success"
-                datax_report_write
+                add_log "D" "cmd: python3 ${DATAX_TOOL_PATH}/bin/datax.py ${conf_dir}/${basename_conf_file}"
+                if ! python3 ${DATAX_TOOL_PATH}/bin/datax.py ${conf_dir}/${basename_conf_file}; then
+                    add_log "E" "Result: failed"
+                    let failed=failed+1
+                else
+                    add_log "I" "Result: success"
+                    datax_report_write
+                fi
+
             fi
+
             let i=i+1
         done
         let total=i-1
@@ -104,28 +164,6 @@ function datax_run()
         add_log "I" "------TOTAL: ${total}, FAILED: ${failed} ------"
         add_log "I" "---------------------------------"
 
-    elif [ -s ${DATAX_CONF_PATH} ]; then
-        failed=0
-        total=1
-        add_log "I" "DATAX_CONF_PATH ${DATAX_CONF_PATH} is a file"
-
-        add_log "I" "Please confirm if you really want to continue to execute datax jobs(Yes/No): "
-        read_user_confirm
-
-        if ! python3 ${DATAX_TOOL_PATH}/bin/datax.py ${DATAX_CONF_PATH}; then
-            add_log "E" "Result: failed"
-            failed=1
-        else
-            add_log "I" "Result: success"
-            datax_report_write
-
-            add_log "I" "---------------------------------"
-            add_log "I" "             SUMMARY             "
-            add_log "I" "------TOTAL: ${total}, FAILED: ${failed} ------"
-            add_log "I" "---------------------------------"
-
-        fi
-
     else
         add_log "E" "DATAX_CONF_PATH ${DATAX_CONF_PATH} is not a valid directory or a non-empty file, exiting"
         failed=1

diff --git a/bin/get_branch.sh b/bin/get_branch.sh
@@ -27,15 +27,18 @@ function get_branch()
     current_branch=`echo "${current_branch:2}"`
 
     cd ${MO_PATH}/matrixone
-    if echo "${current_branch}" | grep "HEAD" >/dev/null 2>&1; then
+    if echo "${current_branch}" | grep "HEAD" >/dev/null 2>&1   || echo "${current_branch}" | grep "detached" >/dev/null 2>&1 ; then
         if [[ "${option}" != "less" ]]; then
             add_log "I" "current_branch is ${current_branch}, contains \"HEAD\" info, thus it's a commit id, trying to find it's real branch"
         fi
         cid_full=`git log | head -n 1 | awk {'print $2'}`
         cid_less=`echo "${cid_full:0:8}"`
         cd ${MO_PATH}/matrixone
-        current_branch=`git branch --contains ${cid_less} -a | grep -v HEAD | sed 's/ //g' | awk -F "/" '{print $NF}'`
-        add_log "D" "cid_full: ${cid_full}, cid_less: ${cid_less}, current_branch: ${current_branch}"
+        current_branch=`git branch --contains ${cid_less} -a | grep -v HEAD | grep -v "detached" | grep "remotes" | sed 's/ //g' | awk -F "/" '{print $NF}'`
+        if [[ "${option}" != "less" ]]; then
+            add_log "D" "cid_full: ${cid_full}, cid_less: ${cid_less}, current_branch: ${current_branch}"
+        fi
+
     fi
 
     if [[ "${current_branch}" != "" ]]; then

diff --git a/bin/help.sh b/bin/help.sh
@@ -541,6 +541,7 @@ function help_datax()
     echo "                  1. DATAX_TOOL_PATH (default: /data/tools/datax): installation path to datax tool"
     echo "                  2. DATAX_CONF_PATH (default: /data/tools/datax/conf/test.json): path to datax conf file, e.g. /data/tools/datax/conf/test.json , or a directory containing multiple conf files, e.g. /data/tools/datax/conf/"
     echo "                  3. DATAX_REPORT_FILE (optional, default:  \${TOOL_LOG_PATH}/backup/report.txt): path to mo_ctl datax report file"
+    echo "                  4. DATAX_PARA_NAME_LIST (optional, default: none): parameter lists of datax -p option, e.g. 'db_name,table_name'"
     echo "Examples        : ${TOOL_NAME} ${option}                        # run datax jobs"
     echo "                : ${TOOL_NAME} ${option} list                   # list datax report"
 }

diff --git a/bin/restore.sh b/bin/restore.sh
@@ -219,7 +219,6 @@ function restore_physical()
 function restore_mo_data()
 {
     add_log "I" "Step_2. Move mo-data path"
-
     if [[ "${MO_DEPLOY_MODE}" == "git" ]]; then
         mo_path="${MO_PATH}/matrixone"
     elif [[ "${MO_DEPLOY_MODE}" == "binary" ]]; then
@@ -229,17 +228,17 @@ function restore_mo_data()
         return 1
     fi
 
+    add_log "D" "MO_DEPLOY_MODE: ${MO_DEPLOY_MODE}, mo_path: ${mo_path}"
     current_time=`date +"%Y%m%d_%H%M%S"`
     if [[ ! -d "${mo_path}/mo-data/" ]]; then
-        add_log "W" "${MO_PATH}/mo-data/ does not exist, skip moving it"
-        return 0
-    fi
-
-    add_log "I" "Moving ${mo_path}/mo-data to ${mo_path}/mo-data-bk-${current_time}"
-    add_log "D" "cmd: mv ${mo_path}/mo-data ${mo_path}/mo-data-bk-${current_time}"
-    if ! mv ${mo_path}/mo-data ${mo_path}/mo-data-bk-${current_time}; then
-        add_log "E" "Failed, exiting"
-        return 1
+        add_log "W" "${mo_path}/mo-data/ does not exist, skip renaming it"
+    else
+        add_log "I" "Renaming ${mo_path}/mo-data to ${mo_path}/mo-data-bk-${current_time}"
+        add_log "D" "cmd: mv ${mo_path}/mo-data ${mo_path}/mo-data-bk-${current_time}"
+        if ! mv ${mo_path}/mo-data ${mo_path}/mo-data-bk-${current_time}; then
+            add_log "E" "Failed, exiting"
+            return 1
+        fi
     fi
 
     add_log "I" "Moving ${RESTORE_PATH}/mo-data to ${mo_path}/mo-data"

diff --git a/bin/start.sh b/bin/start.sh
@@ -110,8 +110,8 @@ function start()
             fi
 
             debug_option=""
-            if [[ "${MO_CONTAINER_DEBUG_PORT}" != "" ]]; then
-                debug_option="-debug-http :${MO_CONTAINER_DEBUG_PORT}"
+            if [[ "${MO_DEBUG_PORT}" != "" ]]; then
+                debug_option="-debug-http :${MO_DEBUG_PORT}"
             fi
 
             pprof_option=""
@@ -157,8 +157,8 @@ function start()
         fi
 
         debug_option=""
-        if [[ "${MO_CONTAINER_DEBUG_PORT}" != "" ]]; then
-            debug_option="-debug-http :${MO_CONTAINER_DEBUG_PORT}"
+        if [[ "${MO_DEBUG_PORT}" != "" ]]; then
+            debug_option="-debug-http :${MO_DEBUG_PORT}"
         fi
 
         pprof_option=""

diff --git a/conf/env.sh b/conf/env.sh
@@ -346,4 +346,6 @@ DATAX_TOOL_PATH="/data/tools/datax"
 DATAX_CONF_PATH="/data/tools/datax/conf/test.json"
 # path to mo_ctl datax report file
 DATAX_REPORT_FILE="${TOOL_LOG_PATH}/datax.report"
+# parameter lists of datax -p option, e.g. 'db_name,table_name'
+DATAX_PARA_NAME_LIST=""