Skip to content

Commit

Permalink
fix: some issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@aronchanisme
aronchanisme committed Jun 13, 2024
1 parent 793f801 commit d3f5dc5
Show file tree
Hide file tree
Showing 10 changed files with 307 additions and 96 deletions.
52 changes: 39 additions & 13 deletions bin/auto_backup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,19 @@ function backup_list()
fi


br_meta_option=""
if [[ "${BACKUP_MOBR_META_PATH}" != "" ]]; then
br_meta_option="--meta_path ${BACKUP_MOBR_META_PATH}"
#add_log "D" "BACKUP_MOBR_META_PATH is not empty, will add option ${br_meta_option}"
fi

#add_log "I" "Listing backup report (detail, physical only)"
#add_log "I" "------------------------------------"
cd ${BACKUP_MOBR_DIRNAME} && ./mo_br list
#add_log "D" "cmd: cd ${BACKUP_MOBR_DIRNAME} && ./mo_br ${br_meta_option} list"
cd ${BACKUP_MOBR_DIRNAME} && ./mo_br ${br_meta_option} list
else
if [[ ! -f ${BACKUP_REPORT} ]]; then
add_log "E" "No backup action can be found, exiting"
add_log "E" "BACKUP_REPORT ${BACKUP_REPORT} is not a valid file, exiting"
return 1
fi
#add_log "I" "Listing backup report (summary) from ${BACKUP_REPORT}"
Expand All @@ -87,15 +94,17 @@ function backup_list()
function backup()
{

! backup_precheck "mo" && return 1
if ! backup_precheck "mo"; then
return 1
fi

add_log "I" "Backup settings"
add_log "I" "------------------------------------"
get_conf | grep BACKUP
add_log "I" "------------------------------------"


add_log "I" "Backup starts"
add_log "I" "Backup begins"

backup_yearmonth=`date '+%Y%m'`
backup_timestamp=`date '+%Y%m%d_%H%M%S'`
Expand All @@ -112,7 +121,7 @@ function backup()
mkdir -p ${backup_report_path}
if [[ ! -f ${BACKUP_REPORT} ]]; then
add_log "D" "Creating backup report file ${BACKUP_REPORT}"
echo "backup_date|backup_target|ds_name|db_list|backup_type|backup_path|logical_data_type|duration_ms|outcome|size_in_bytes" > "${BACKUP_REPORT}"
echo "backup_date|backup_target|ds_name|db_list|backup_type|backup_path|logical_data_type|duration_ms|outcome|bk_size_in_bytes|logical_net_buffer_length" > "${BACKUP_REPORT}"
fi

backup_db_list=""
Expand All @@ -124,8 +133,11 @@ function backup()
# 1) logical backups : mo_dump
"logical")
logical_data_type=${BACKUP_LOGICAL_DATA_TYPE}
net_buffer_length="${BACKUP_LOGICAL_NETBUFLEN}"

! backup_precheck "modump" && return 1
if ! backup_precheck "modump";then
return 1
fi

all_dbs=`MYSQL_PWD="${MO_PW}" mysql -u"${MO_USER}" -P"${MO_PORT}" -h"${MO_HOST}" -e "show databases" -N -s`
add_log "D" "All databases in current system: ${all_dbs}"
Expand Down Expand Up @@ -286,17 +298,25 @@ function backup()
#backup_db_list="all"
backup_conf_db_list="all"
logical_data_type="n.a."
net_buffer_length="n.a."

! backup_precheck "mobr" && return 1
if ! backup_precheck "mobr"; then
return 1
fi

br_meta_option=""
if [[ "${BACKUP_MOBR_META_PATH}" != "" ]]; then
br_meta_option="--meta_path ${BACKUP_MOBR_META_PATH}"
add_log "D" "BACKUP_MOBR_META_PATH is not empty, will add option ${br_meta_option}"
fi

BACKUP_MOBR_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"`
case "${BACKUP_PHYSICAL_TYPE}" in
"filesystem")
add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"filesystem\" --path \"${backup_outpath}/\""
add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"filesystem\" --path \"${backup_outpath}/\""

startTime=`get_nanosecond`
if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "filesystem" --path "${backup_outpath}/" ; then
if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "filesystem" --path "${backup_outpath}/" ; then
outcome="succeeded"
else
outcome="failed"
Expand All @@ -305,7 +325,7 @@ function backup()
;;
"s3")
minio_option=""
if [[ "${BACKUP_S3_IS_MINIO}" != "no" ]]; then
if [[ "${BACKUP_S3_IS_MINIO}" == "yes" ]]; then
minio_option="--is_minio"
fi

Expand All @@ -314,10 +334,10 @@ function backup()
role_arn_option="--role_arn ${BACKUP_S3_ROLE_ARN}"
fi

add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"s3\" --endpoint \"${BACKUP_S3_ENDPOINT}\" --access_key_id \"${BACKUP_S3_ID}\" --secret_access_key \"${BACKUP_S3_KEY}\" --bucket \"${BACKUP_S3_BUCKET}\" --filepath \"${BACKUP_DATA_PATH}\" --region \"${BACKUP_S3_REGION}\" --compression \"${BACKUP_S3_COMPRESSION}\" \"${role_arn_option}\" \"${minio_option}\""
add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"s3\" --endpoint \"${BACKUP_S3_ENDPOINT}\" --access_key_id \"${BACKUP_S3_ID}\" --secret_access_key \"${BACKUP_S3_KEY}\" --bucket \"${BACKUP_S3_BUCKET}\" --filepath \"${BACKUP_DATA_PATH}\" --region \"${BACKUP_S3_REGION}\" --compression \"${BACKUP_S3_COMPRESSION}\" \"${role_arn_option}\" \"${minio_option}\""

startTime=`get_nanosecond`
if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "s3" --endpoint "${BACKUP_S3_ENDPOINT}" --access_key_id "${BACKUP_S3_ID}" --secret_access_key "${BACKUP_S3_KEY}" --bucket "${BACKUP_S3_BUCKET}" --filepath "${BACKUP_DATA_PATH}" --region "${BACKUP_S3_REGION}" --compression "${BACKUP_S3_COMPRESSION}" "${role_arn_option}" "${minio_option}"; then
if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "s3" --endpoint "${BACKUP_S3_ENDPOINT}" --access_key_id "${BACKUP_S3_ID}" --secret_access_key "${BACKUP_S3_KEY}" --bucket "${BACKUP_S3_BUCKET}" --filepath "${BACKUP_DATA_PATH}" --region "${BACKUP_S3_REGION}" --compression "${BACKUP_S3_COMPRESSION}" "${role_arn_option}" "${minio_option}"; then
outcome="succeeded"
else
outcome="failed"
Expand All @@ -337,13 +357,19 @@ function backup()
add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms"

esac


bk_size="n.a."
if [[ ${outcome} == "succeeded" ]]; then
add_log "D" "Calculating size of backup path ${backup_outpath}"
bk_size=`du -s ${backup_outpath} | awk '{print $1}'`
fi

# output report record
add_log "D" "Writing entry to report ${BACKUP_REPORT}"
add_log "D" "${backup_timestamp}|${bakcup_target}|${BACKUP_LOGICAL_DS}|${backup_conf_db_list}|${BACKUP_TYPE}|${backup_outpath}|${logical_data_type}|${cost}|${outcome}|${bk_size}|${net_buffer_length}"
bakcup_target="${MO_HOST},${MO_PORT},${MO_USER}"
echo "${backup_timestamp}|${bakcup_target}|${BACKUP_LOGICAL_DS}|${backup_conf_db_list}|${BACKUP_TYPE}|${backup_outpath}/${backup_yearmonth}/${backup_timestamp}/|${logical_data_type}|${cost}|${outcome}|${bk_size}" >> "${BACKUP_REPORT}"
echo "${backup_timestamp}|${bakcup_target}|${BACKUP_LOGICAL_DS}|${backup_conf_db_list}|${BACKUP_TYPE}|${backup_outpath}|${logical_data_type}|${cost}|${outcome}|${bk_size}|${net_buffer_length}" >> "${BACKUP_REPORT}"

if [[ ${rc} -ne 0 ]]; then
add_log "E" "Backup ends with non-zero rc"
Expand Down
28 changes: 22 additions & 6 deletions bin/csv_convert.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,20 +421,36 @@ function csv_convert()
case "${CSV_CONVERT_TYPE}" in
"1")
convert_type="insert"
! csv_convert_prep ${convert_type} && return 1
! csv_convert_insert_${run_mode} && return 1
if ! csv_convert_prep ${convert_type}; then
return 1
fi

if ! csv_convert_insert_${run_mode}; then
return 1
fi

;;
"2")
line_mode="single"
convert_type="load-singleline"
! csv_convert_prep ${convert_type} && return 1
! csv_convert_load_${run_mode} "${line_mode}" && return 1
if ! csv_convert_prep ${convert_type}; then
return 1
fi

if ! csv_convert_load_${run_mode} "${line_mode}"; then
return 1
fi
;;
"3")
line_mode="multi"
convert_type="load-multilines"
! csv_convert_prep ${convert_type} && return 1
! csv_convert_load_${run_mode} "${line_mode}" && return 1
if ! csv_convert_prep ${convert_type}; then
return 1
fi

if ! csv_convert_load_${run_mode} "${line_mode}"; then
return 1
fi
;;
*)
return 1
Expand Down
20 changes: 13 additions & 7 deletions bin/help.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,21 +55,27 @@ function help_precheck()
function help_deploy()
{
option="deploy"
echo "Usage : ${TOOL_NAME} ${option} [mo_version] [force] # ${USAGE_DEPLOY}"
echo "Usage : ${TOOL_NAME} ${option} [\$mo_version] [force] [nobuild] # ${USAGE_DEPLOY}"
echo " : ${TOOL_NAME} ${option} help # print help info"
echo " [mo_version]: optional: specify an mo version to deploy"
echo " [\$mo_version]: optional: specify an mo version to deploy"
echo " [force] : optional: if specified will delete all content under MO_PATH and deploy from beginning"
echo " [nobuild] : optional: if specified will skip building mo-service"
echo " Note : 'deploy' is valid only when MO_DEPLOY_MODE is set to 'git' or 'docker'"
echo " e.g. : ${TOOL_NAME} ${option} # default, same as ${TOOL_NAME} ${option} ${MO_DEFAULT_VERSION}"
echo " : ${TOOL_NAME} ${option} main # deploy development latest version"
echo " : ${TOOL_NAME} ${option} d29764a # deploy development version d29764a"
echo " : ${TOOL_NAME} ${option} 0.8.0 # deploy stable verson 0.8.0"
echo " : ${TOOL_NAME} ${option} 1.2.0 # deploy stable verson 1.2.0"
echo " : ${TOOL_NAME} ${option} force # delete all under MO_PATH and deploy verson ${MO_DEFAULT_VERSION}"
echo " : ${TOOL_NAME} ${option} 0.8.0 force # delete all under MO_PATH and deploy stable verson 0.8.0 from beginning"
echo " : ${TOOL_NAME} ${option} 1.2.0 force # delete all under MO_PATH and deploy stable verson 1.2.0 from beginning"
}



function help_deploy_docker()
{
option="deploy"
echo "Usage : ${TOOL_NAME} ${option} # ${USAGE_DEPLOY}"
echo " : ${TOOL_NAME} ${option} help # print help info"
}

function help_status()
{
Expand Down Expand Up @@ -203,11 +209,11 @@ function help_upgrade()
option="upgrade"
echo "Usage : ${TOOL_NAME} ${option} [version_or_commitid] # ${USAGE_UPGRADE}"
echo " : ${TOOL_NAME} ${option} help # print help info"
echo " [commitid] : a commit id such as '38888f7', or a stable version such as '0.8.0'"
echo " [commitid] : a commit id such as '38888f7', or a stable version such as '1.2.0'"
echo " : use 'latest' to upgrade to latest commit on main branch if you don't know the id"
echo " e.g. : ${TOOL_NAME} ${option} 38888f7 # upgrade/downgrade to commit id 38888f7 on main branch"
echo " : ${TOOL_NAME} ${option} latest # upgrade/downgrade to latest commit on main branch"
echo " : ${TOOL_NAME} ${option} 0.8.0 # upgrade/downgrade to stable version 0.8.0"
echo " : ${TOOL_NAME} ${option} 1.2.0 # upgrade/downgrade to stable version 1.2.0"

}

Expand Down
Loading

0 comments on commit d3f5dc5

Please sign in to comment.