unit_test_linux_ubuntu.yaml

name: Unit Test (Linux-Ubuntu)

run-name: Unit Test (Linux-Ubuntu)

on: 
  push:
    branches:
    #  - main
      - dev
#  schedule:
    # run every night at 00:00
#    - cron: '0 0 * * *'
  workflow_dispatch:
   inputs:
      RUN_OR_NOT:
         description: 'Run or not?'
         required: true
         default: true
         type: boolean

# global vars
env:
  MO_CTL_GIT_URL: "https://raw.githubusercontent.com/matrixorigin/mo_ctl_standalone/main/"
  GO_PROXY: "https://proxy.golang.org"
  MO_HOST: "127.0.0.1"
  MO_PORT: "6001"
  MO_USER: "dump"
  MO_PW: "111"
  OPTION_LIST: "auto_backup,auto_clean_logs,backup,clean_backup,clean_logs,connect,ddl_convert,deploy,get_branch,get_cid,get_conf,pprof,precheck,restart,set_conf,start,status,stop,uninstall,upgrade,watchdog"
  MO_PATH: "/home/runner/mo"
  MO_STABLE_VERSION: "v1.1.0"
  MO_STABLE_BRANCH: "(HEAD detached at v1.1.0)"
  # last updated: 2023/10/9
  MO_STABLE_CID: "daf86797160585d24c158e4ee220964264616505"
  MO_TEST_CID_1: "f2ae76f49"
  MO_TEST_BRANCH_1: "(HEAD detached at f2ae76f49)"
  MO_TEST_CID_2: "9d5c72b12"
  MO_TEST_BRANCH_2: "(HEAD detached at 9d5c72b12)"
  MO_1_1_DEV_BRANCH: "1.1-dev"
  MO_MAIN_BRANCH: "main"
  MO_PPROF_PATH: "/home/runner/pprof_test"
  TEST_SQL_PATH: "/home/runner/sql_test_path"
  TEST_SQL_FILE_1: "${TEST_SQL_PATH}/q1.sql"
  TEST_SQL_FILE_2: "${TEST_SQL_PATH}/q2.sql"
  TEST_SQL_FILE_3: "${TEST_SQL_PATH}/q3.sql"
  MO_GIT_URL_1: https://mirror.ghproxy.com/github.com/matrixorigin/matrixone.git
  MO_GIT_URL_2: https://mirror.ghproxy.com/github.com/matrixorigin/matrixone.git
  BACKUP_PATH: "/home/runner/mo-backup"
  MO_TOOL_PATH: "/home/runner/tools"
  MO_TOOL_URL: "https://mo-tools-1308875761.cos.ap-chengdu.myqcloud.com/latest"
  MO_CTL_TOOL_LOG_PATH: "/home/runner/logs/mo_ctl"

# job def
jobs:
  Job_01:
    name: Job_01 - install | uninstall mo_ctl
    runs-on: ubuntu-latest
    steps:
      - name: Test_1. 'install.sh'
        run: |
          echo "Step_1. install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mo_ctl help
          
          echo ""
          echo "Step_2. install mo_ctl with Internet(proxy)"
          wget https://mirror.ghproxy.com/https://github.com/matrixorigin/mo_ctl_standalone/blob/main/install.sh && bash +x install.sh
          mo_ctl help
          
          echo ""
          echo "Step_3. install mo_ctl without Internet"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh
          wget https://github.com/matrixorigin/mo_ctl_standalone/archive/refs/heads/main.zip -O mo_ctl.zip
          bash +x ./install.sh mo_ctl.zip
          mo_ctl help
      - name: Test_2. 'uninstall.sh'
        run: |
          echo "Step_1. uninstall mo_ctl"
          wget ${{ env.MO_CTL_GIT_URL }}/uninstall.sh && echo "yes" | bash +x ./uninstall.sh
          
          echo ""
          echo "Step_2. check if mo_ctl has been uninstalled"
          
          if which mo_ctl; then
              echo "mo_ctl found in PATH: $PATH, Failed"
              exit 1
          else
              echo "No mo_ctl found in PATH: $PATH"
          fi
          
          if [[ -d ~/mo_ctl/ ]]; then
              echo "Path ~/mo_ctl/ is not removed, Failed"
              ls -lth ~/mo_ctl/
              exit 1
          else
              echo "Path ~/mo_ctl/ has been removed"
          fi

  Job_02:
    name: Job_02 - help
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
      - name: Test_1. 'mo_ctl help' (show usage)
        run: |
          echo ""
          mo_ctl help
      - name: Test_2. 'mo_ctl [option_1] help' (valid option_1)
        run: |
          i=1
          for option in $(echo ${{ env.OPTION_LIST }} | sed "s/,/ /g"); do
              echo ""
              echo "--------------------------------------------------------------------"
              echo "Loop number: ${i}, option: ${option}, command: mo_ctl ${option} help"
              mo_ctl ${option} help
              ((i++))
          done
      - name: Test_3. 'mo_ctl xxx' (invalid option_1)
        run: |
          if mo_ctl xxx; then
              echo "'mo_ctl xxx' should fail but it didn't"
              exit 1
          fi
      - name: Test_4. 'mo_ctl xxx help' (invalid option_1 help)
        run: |
          if mo_ctl xxx help; then
              echo "'mo_ctl xxx help' should fail but it didn't"
              exit 1
          fi
  Job_03:
    name: Job_03 - set_conf | get_conf
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
      - name: Test_1. 'mo_ctl get_conf [all]' (get all confs)
        run: |
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
          echo ""
          echo "test_cmd: mo_ctl get_conf"
          mo_ctl get_conf
          echo ""
          echo "test_cmd: mo_ctl get_conf all"
          mo_ctl get_conf all
      - name: Test_2. 'mo_ctl get_conf MO_PORT' (get single conf)
        run: |
          echo ""
          echo "test_cmd: mo_ctl get_conf MO_PROT"
          mo_ctl get_conf MO_PORT
      - name: Test_3. 'mo_ctl get_conf MO_USER,MO_PW' (get multiple conf)
        run: |
          echo ""
          echo "test_cmd: mo_ctl get_conf MO_USER,MO_PW"
          mo_ctl get_conf MO_USER,MO_PW
      - name: Test_4. 'mo_ctl get_conf xxx' (wrong way to get conf)
        run: |
          echo ""
          echo "test_cmd: mo_ctl get_conf xx || mo_ctl get_conf MO_USER,yy || mo_ctl get_conf MO_USER=yy"
          if mo_ctl get_conf xx || mo_ctl get_conf MO_USER,yy || mo_ctl get_conf MO_USER=yy; then
              echo "'mo_ctl get_conf' does not fail as expected"
              exit 1
          fi
      - name: Test_5. 'mo_ctl set_conf MO_USER' (set single conf)
        run: |
          echo ""
          echo "test_cmd: mo_ctl set_conf MO_USER=\"mike\""
          mo_ctl set_conf MO_USER="mike"
          mo_ctl get_conf MO_USER | grep "mike" 
      #- name: Test_6. 'mo_ctl set_conf MO_USER="mike",MO_PW="M@trix0riginRocks"' (set multiple conf)
      #  run: |
      #    echo ""
          #echo "test_cmd: mo_ctl set_conf MO_USER=\"mike\",MO_PW=\"M@trix0riginRocks\""
          #mo_ctl set_conf MO_USER="mike",MO_PW="M@trix0riginRocks"
          #mo_ctl get_conf MO_USER | grep "mike" 
          #mo_ctl get_conf MO_PW | grep "M@trix0riginRocks" 
      #- name: Test_7. 'mo_ctl set_conf MO_USER="mike" MO_PW="M@trix0riginRocks"' (set multiple conf)
      #  run: |
      #    echo ""
          #echo "test_cmd: mo_ctl set_conf MO_USER=\"mike\" MO_PW=\"M@trix0riginRocks\""
          #mo_ctl set_conf MO_USER="mike" MO_PW="M@trix0riginRocks"
          #mo_ctl get_conf MO_USER | grep "mike" 
          #mo_ctl get_conf MO_PW | grep "M@trix0riginRocks" 
      - name: Test_6. 'mo_ctl set_conf xxx' (wrong way to set conf)
        run: |
          echo ""
          #echo "test_cmd: mo_ctl set_conf xx || mo_ctl set_conf MO_USER=\"mike\",yy || mo_ctl set_conf MO_USER=\"mike\",yy=\"test\" || mo_ctl set_conf MO_USER=\"mike\" yy=\"test\""
          echo "test_cmd: mo_ctl set_conf xx || mo_ctl set_conf yy=\"test\""
          #if mo_ctl set_conf xx || mo_ctl set_conf MO_USER="mike",yy ||mo_ctl set_conf MO_USER="mike",yy="test" || mo_ctl set_conf MO_USER="mike" yy="test"; then
          if mo_ctl set_conf xx || mo_ctl set_conf yy="test"; then
              echo "'mo_ctl set_conf' does not fail as expected"
              exit 1
          fi
      - name: Test_9. 'mo_ctl set_conf MO_GIT_URL' (special settings)
        run: |
          echo ""
          echo "test_cmd: mo_ctl set_conf ${{ env.MO_GIT_URL_1 }}"
          if mo_ctl set_conf ${{ env.MO_GIT_URL_1 }}; then
              echo "failed"
              exit 1
          fi
          echo "test_cmd: mo_ctl set_conf ${{ env.MO_GIT_URL_2 }}"
          if mo_ctl set_conf ${{ env.MO_GIT_URL_2 }}; then
              echo "failed"
              exit 1
          fi          
      - name: Test_10. 'mo_ctl set_conf reset' (reset all confs to default)
        run: |
          echo ""
          echo "test_cmd: echo \"yes\" | mo_ctl set_conf reset"
          echo "yes"| mo_ctl set_conf reset

  Job_04:
    name: Job_04 - precheck
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
      - name: Test_1. 'mo_ctl precheck' (when precheck succeeds)
        run: |
          echo ""
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
          mo_ctl precheck
          echo "'mo_ctl precheck' succeeds as expected"
      - name: Test_2. 'mo_ctl precheck' (when precheck fails)
        run: |
          echo ""
          echo "Removing mysql client"
          mysql_file=`which mysql`
          sudo rm -f ${mysql_file}
          if ! mo_ctl precheck; then
              echo "'mo_ctl precheck' fails as expected"
          fi
  Job_05:
    name: Job_05 - get_branch | get_cid | version
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy main' (deploy latest commit id on main)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy main"
          mo_ctl deploy main
      - name: Test_2. 'mo_ctl get_branch' (main)
        run: |
          mo_ctl get_branch
          branch=`mo_ctl get_branch | tail -n 1 | awk -F'current branch: ' '{print $2}'`
          echo "branch is ${branch}"
          if [[ "${branch}" != "${{ env.MO_MAIN_BRANCH }}" ]]; then
              echo "Failed to get_branch or branch is not that as expected"
              exit 1
          fi
      - name: Test_3. 'mo_ctl get_cid' (get commit id)
        run: |
          echo "------------------------------"
          echo "Commit id (full):"
          mo_ctl get_cid

          echo "------------------------------"
          echo "Commit id (less):"
          mo_ctl get_cid less

      - name: Test_4. 'mo_ctl version' (get mo_ctl and mo version)
        run: |
          echo ""
          mo_ctl version
      - name: Test_5. 'mo_ctl uninstall' (uninstall mo)
        run: |
          echo ""
          mo_ctl stop force
          echo "yes" | mo_ctl uninstall
          if [[ -d "${{ env.MO_PATH }}/${RUN_TAG}/matrixone" ]]; then
              echo "Uninstall failed"
              exit 1
          fi

  Job_06:
    name: Job_06 - deploy | uninstall (stable)
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy [default_version]' (deploy current stable version)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy ${{ env.MO_STABLE_VERSION }}"
          mo_ctl deploy ${{ env.MO_STABLE_VERSION }}
      - name: Test_2. 'mo_ctl get_branch' (get current branch)
        run: |
          mo_ctl get_branch
          branch=`mo_ctl get_branch | tail -n 1 | awk -F'current branch: ' '{print $2}'`
          echo "branch is: ${branch}"
          if [[ "${branch}" != "${{ env.MO_STABLE_BRANCH }}" ]]; then
              echo "Failed to get_branch or branch is not that as expected: ${{ env.MO_STABLE_BRANCH }}"
              echo "expected branch: ${{ env.MO_STABLE_BRANCH }}, actual branch: ${${branch}}"
              exit 1
          fi
      - name: Test_3. 'mo_ctl get_cid' (get commit id)
        run: |
          mo_ctl get_cid
          cid=`mo_ctl get_cid less | sed -n '2p'`
          if [[ "${cid}" == "" ]]; then
              echo "Failed to get_cid or cid is not that as expected"
              exit 1
          fi
      - name: Test_4. 'mo_ctl uninstall' (uninstall mo)
        run: |
          echo ""
          mo_ctl stop force
          echo "yes" | mo_ctl uninstall
          if [[ -d "${{ env.MO_PATH }}/${RUN_TAG}/matrixone" ]]; then
              echo "Uninstall failed"
              exit 1
          fi

  Job_07:
    name: Job_07 - deploy | uninstall (main-latest)
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy main' (deploy main latest commit)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy ${{ env.MO_MAIN_BRANCH }}"
          mo_ctl deploy ${{ env.MO_MAIN_BRANCH }}
      - name: Test_2. 'mo_ctl get_branch' (get current branch)
        run: |
          mo_ctl get_branch
          branch=`mo_ctl get_branch | tail -n 1 | awk -F'current branch: ' '{print $2}'`
          echo "branch is: ${branch}"
          if [[ "${branch}" != "${{ env.MO_MAIN_BRANCH }}" ]]; then
              echo "Failed to get_branch or branch is not that as expected: ${{ env.MO_MAIN_BRANCH }}"
              echo "expected branch: ${{ env.MO_MAIN_BRANCH }}, actual branch: ${branch}"
              exit 1
          fi
      - name: Test_3. 'mo_ctl get_cid' (get commit id)
        run: |
          mo_ctl get_cid
          cid=`mo_ctl get_cid less | sed -n '2p'`
          if [[ "${cid}" == "" ]]; then
              echo "Failed to get_cid or cid is not that as expected"
              exit 1
          fi
      - name: Test_4. 'mo_ctl uninstall' (uninstall mo)
        run: |
          echo ""
          mo_ctl stop force
          echo "yes" | mo_ctl uninstall
          if [[ -d "${{ env.MO_PATH }}/${RUN_TAG}/matrixone" ]]; then
              echo "Uninstall failed"
              exit 1
          fi

  Job_08:
    name: Job_08 - deploy | uninstall (main - commitid)
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy [cid_1]' (deploy main cid_1 commit)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy ${{ env.MO_TEST_CID_1 }}"
          mo_ctl deploy ${{ env.MO_TEST_CID_1 }}
      - name: Test_2. 'mo_ctl get_branch' (get current branch)
        run: |
          mo_ctl get_branch
          branch=`mo_ctl get_branch | tail -n 1 | awk -F'current branch: ' '{print $2}'`
          echo "branch is: ${branch}"
          if [[ "${branch}" != "${{ env.MO_TEST_BRANCH_1 }}" ]]; then
              echo "Failed to get_branch or branch is not that as expected: ${{ env.MO_TEST_BRANCH_1 }}"
              echo "expected branch: ${{ env.MO_TEST_BRANCH_1 }}, actual branch: ${${branch}}"
              exit 1
          fi
      - name: Test_3. 'mo_ctl get_cid' (get commit id)
        run: |
          mo_ctl get_cid
          cid=`mo_ctl get_cid less | sed -n '2p'`
          if [[ "${cid}" == "" ]]; then
              echo "Failed to get_cid or cid is not that as expected"
              exit 1
          fi
      - name: Test_4. 'mo_ctl uninstall' (uninstall mo)
        run: |
          echo ""
          mo_ctl stop force
          echo "yes" | mo_ctl uninstall force
          if [[ -d "${{ env.MO_PATH }}/${RUN_TAG}/matrixone" ]]; then
              echo "Uninstall failed"
              exit 1
          fi

  Job_09:
    name: Job_09 - deploy | uninstall (1.1-dev - latest)
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy main' (deploy main latest commit)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy ${{ env.MO_1_1_DEV_BRANCH }}"
          mo_ctl deploy ${{ env.MO_1_1_DEV_BRANCH }}
      - name: Test_2. 'mo_ctl get_branch' (get current branch)
        run: |
          mo_ctl get_branch
          branch=`mo_ctl get_branch | tail -n 1 | awk -F'current branch: ' '{print $2}'`
          echo "branch is: ${branch}"
          if [[ "${branch}" != "${{ env.MO_1_1_DEV_BRANCH }}" ]]; then
              echo "Failed to get_branch or branch is not that as expected: ${{ env.MO_1_1_DEV_BRANCH }}"
              echo "expected branch: ${{ env.MO_1_1_DEV_BRANCH }}, actual branch: ${${branch}}"
              exit 1
          fi
      - name: Test_3. 'mo_ctl get_cid' (get commit id)
        run: |
          mo_ctl get_cid
          cid=`mo_ctl get_cid less | sed -n '2p'`
          if [[ "${cid}" == "" ]]; then
              echo "Failed to get_cid or cid is not that as expected"
              exit 1
          fi
      - name: Test_4. 'mo_ctl uninstall' (uninstall mo)
        run: |
          echo ""
          mo_ctl stop force
          echo "yes" | mo_ctl uninstall
          if [[ -d "${{ env.MO_PATH }}/${RUN_TAG}/matrixone" ]]; then
              echo "Uninstall failed"
              exit 1
          fi

  Job_10:
    name: Job_10 - build_image
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}

      - name: Install docker
        uses: docker-practice/actions-setup-docker@master
        timeout-minutes: 12
      - name: Test_1. 'mo_ctl deploy main' (deploy latest)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          mo_ctl set_conf MO_BUILD_IMAGE_PATH=/tmp

          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy main"
          mo_ctl deploy main
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. build image
        run: |
          echo ""
          mo_ctl build_image
      - name: Test_3. check image
        run: |
          echo ""
          docker images
          ls -lth /tmp/

  Job_11:
    name: Job_11 - deploy | uninstall (main - invalid commitid)
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy 12345678' (deploy an invalid commit id)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy 12345678"
          if mo_ctl deploy 12345678; then
              echo "Deploy should fail when deploying an invalid commit id, but it didn't"
              exit 1
          fi

  Job_12:
    name: Job_12 - status | start | stop | restart
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}

          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy main"
          mo_ctl deploy main
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_1. 'mo_ctl status' (no mo-service running)
        run: |
          echo ""
          if ! mo_ctl status; then 
              echo "'mo_ctl status' fails as expected"
          fi
      - name: Test_2. 'mo_ctl start'
        run: |
          echo ""
          mo_ctl start && mo_ctl status
      - name: Test_3. 'mo_ctl status' (mo-service running)
        run: |
          echo ""
          mo_ctl status
          echo "'mo_ctl status' succeeds as expected"
      - name: Test_4. 'mo_ctl stop'
        run: |
          echo ""
          mo_ctl stop
          if ! mo_ctl status; then 
              echo "Stop as expected"
          fi
      - name: Test_5. 'mo_ctl restart'
        run: |
          echo ""
          mo_ctl restart && mo_ctl status
      - name: Test_6. 'mo_ctl stop force'
        run: |
          echo ""
          mo_ctl stop force
          if ! mo_ctl status; then 
              echo "Stop force as expected"
          fi
      - name: Test_7. 'mo_ctl restart'
        run: |
          echo ""
          mo_ctl restart force && mo_ctl status

  Job_13: 
    name: Job_13 - connect
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}

          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy main"
          mo_ctl deploy main
          echo "Sleep 5 seconds and start mo-service"
          sleep 5 && mo_ctl start
          echo "Check mo-service status"
          mo_ctl status
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_1. 'mo_ctl connect' (mo-service running)
        run: |
          echo ""
          echo "select version()" | mo_ctl connect
      - name: Test_2. 'mo_ctl connect' (mo-service not running)
        run: |
          echo ""
          mo_ctl stop force
          if mo_ctl status; then
              echo "Failed to stop mo-service"
              exit 1
          fi
          if echo "select version()" | mo_ctl connect; then
              echo "When mo-service is not running, mo_ctl connect should fail but it didn't"
              exit 1
          fi

  Job_14: 
    name: Job_14 - sql
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}

          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy main"
          mo_ctl deploy main
          echo "Sleep 5 seconds and start mo-service"
          sleep 5 && mo_ctl start
          echo "Check mo-service status"
          mo_ctl status
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          echo "Create test path and test files"
          mkdir -p ${{ env.TEST_SQL_PATH }}
          echo "select version();" > ${{ env.TEST_SQL_FILE_1 }}
          echo "select 1;" > ${{ env.TEST_SQL_FILE_2 }}
          echo "select @@sql_mode;" > ${{ env.TEST_SQL_FILE_3 }}     
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_1. 'mo_ctl sql "[sql_query]"' (valid)
        run: |
          echo ""
          mo_ctl sql "select version()"
      - name: Test_2. 'mo_ctl sql "[sql_query]"' (invalid)
        run: |
          echo ""
          if mo_ctl sql "select xx from xx;"; then
              echo "Invalid sql query does not fail as expected"
              exit 1
          fi
      - name: Test_3. 'mo_ctl sql "[sql_file]"' (valid)
        run: |
          mo_ctl sql ${{ env.TEST_SQL_FILE_1 }}
      - name: Test_4. 'mo_ctl sql "[sql_file]"' (invalid)
        run: |
          echo ""
          if mo_ctl sql /tmp/123/456/789.txt; then
              echo "Invalid sql file does not fail as expected"
              exit 1
          fi
      - name: Test_5. 'mo_ctl sql "[sql_path]"' (valid)
        run: |
          mo_ctl sql ${{ env.TEST_SQL_PATH }}
      - name: Test_6. 'mo_ctl sql "[sql_path]"' (invalid)
        run: |
          if mo_ctl sql /tmp/123/456/789/; then
              echo "Invalid sql path does not fail as expected"
              exit 1
          fi
      - name: Test_7. 'mo_ctl sql ""' (empty input)
        run: |
          if mo_ctl sql || mo_sql sql ""; then
              echo "Invalid sql path does not fail as expected"
              exit 1
          fi

  Job_15: 
    name: Job_15 - watchdog
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy main"
          mo_ctl deploy main
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_1. 'mo_ctl watchdog' (check disabled status)
        run: |
          echo ""
          if mo_ctl watchdog || mo_ctl watchdog status; then
              echo "mo_ctl watchdog does not fail as expected"
              exit 1
          fi
      - name: Test_2. 'mo_ctl watchdog enable' (enable watchdog)
        run: |
          echo ""
          if ! mo_ctl watchdog enable; then
              echo "'mo_ctl watchdog enable' failed"
              exit 1
          fi
          echo "Sleep 65 seconds and see if mo-service is running"
          sleep 65
          mo_ctl status
      - name: Test_3. 'mo_ctl watchdog' (check enabled status)
        run: |
          echo ""
          if mo_ctl watchdog && mo_ctl watchdog status; then
              echo "'mo_ctl watchdog' succeeds as expected"
          else
              echo "'mo_ctl watchdog' failed"
              exit 1
          fi
      - name: Test_4. 'mo_ctl watchdog disable' (disable watchdog)
        run: |
          echo "Stop mo-service"
          if ! mo_ctl stop; then
              mo_ctl stop force
          fi
          echo "Disable mo watchdog"
          mo_ctl watchdog disable
          echo "Check mo_watchdog stauts"
          if mo_ctl watchdog || mo_ctl watchdog status; then
              echo "mo_ctl watchdog does not fail as expected"
              exit 1
          fi
          echo "Sleep 65 seconds and see if mo-service will not be pulled up running"
          sleep 65
          if mo_ctl status; then
              echo "mo-service is still in running status, seems like disabling watchdog did not succeed as expected"
              exit 1
          fi

  Job_16: 
    name: Job_16 - pprof
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}

          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy main"
          mo_ctl deploy main
          echo "Sleep 5 seconds and start mo-service"
          sleep 5 && mo_ctl start
          echo "Check mo-service status"
          mo_ctl status
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          mkdir -p ${{ env.MO_PPROF_PATH }}
          mo_ctl set_conf PPROF_OUT_PATH="${{ env.MO_PPROF_PATH }}"
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_1. 'mo_ctl pprof' (collect profile for default 30 seconds)
        run: |
          echo ""
          mo_ctl pprof
          ls -lth ${{ env.MO_PPROF_PATH }}/
          if [[ "`ls ${{ env.MO_PPROF_PATH }}/ | wc -l | sed s/[[:space:]]//g`" != "1" ]]; then
              echo "No file is generated in ${{ env.MO_PPROF_PATH }}/"
              exit 1
          fi
          cd ${{ env.MO_PPROF_PATH }}/ && rm -f ./*
      - name: Test_2. 'mo_ctl pprof profile' (collect profile for default 30 seconds)
        run: |
          echo ""
          mo_ctl pprof profile
          ls -lth ${{ env.MO_PPROF_PATH }}/
          if [[ "`ls ${{ env.MO_PPROF_PATH }}/ | wc -l | sed s/[[:space:]]//g`" != "1" ]]; then
              echo "No file is generated in ${{ env.MO_PPROF_PATH }}/"
              exit 1
          fi
          cd ${{ env.MO_PPROF_PATH }}/ && rm -f ./*
      - name: Test_3. 'mo_ctl pprof profile 15' (collect profile for 15 seconds)
        run: |
          echo ""
          mo_ctl pprof profile 15
          ls -lth ${{ env.MO_PPROF_PATH }}/
          if [[ "`ls ${{ env.MO_PPROF_PATH }}/ | wc -l | sed s/[[:space:]]//g`" != "1" ]]; then
              echo "No file is generated in ${{ env.MO_PPROF_PATH }}/"
              exit 1
          fi
          cd ${{ env.MO_PPROF_PATH }}/ && rm -f ./*
      - name: Test_4. 'mo_ctl pprof heap' (collect heap)
        run: |
          echo ""
          mo_ctl pprof heap
          ls -lth ${{ env.MO_PPROF_PATH }}/
          if [[ "`ls ${{ env.MO_PPROF_PATH }}/ | wc -l | sed s/[[:space:]]//g`" != "1" ]]; then
              echo "No file is generated in ${{ env.MO_PPROF_PATH }}/"
              exit 1
          fi
          cd ${{ env.MO_PPROF_PATH }}/ && rm -f ./*
      - name: Test_5. 'mo_ctl pprof allocs' (collect allocs)
        run: |
          echo ""
          mo_ctl pprof allocs
          ls -lth ${{ env.MO_PPROF_PATH }}/
          if [[ "`ls ${{ env.MO_PPROF_PATH }}/ | wc -l | sed s/[[:space:]]//g`" != "1" ]]; then
              echo "No file is generated in ${{ env.MO_PPROF_PATH }}/"
              exit 1
          fi
          cd ${{ env.MO_PPROF_PATH }}/ && rm -f ./*

  Job_17:
    name: Job_17 - upgrade (cid_1->cid_2)
    # DO NOT RUN
    if: "${{ inputs.NOT_EXIST_VAR }}"
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy [cid_1]' (deploy cid_1)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy ${{ env.MO_TEST_CID_1 }}"
          mo_ctl deploy ${{ env.MO_TEST_CID_1 }}
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. 'mo_ctl start' (start mo-service) and prepare some data
        run: |
          mo_ctl start
          echo "Sleeping 30 seconds for mo to be ready"
          sleep 30
          echo "Prepare some data"
          mo_ctl sql "create database test; use test; create table tb1(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"
      - name: Test_3. 'mo_ctl upgrade [cid_2]' (cid_1 -> cid_2)
        run: |
          if ! mo_ctl stop; then
              mo_ctl stop force
          fi
          sleep 5
          echo "Sleep 5 seconds before upgrading"
          mo_ctl upgrade ${{ env.MO_TEST_CID_2 }}
          mo_ctl start
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          if ! mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"; then
              echo "Run query failed. This may be due to incompatible data format when upgrading from one cid to another"
          fi

  Job_18:
    name: Job_18 - upgrade (cid_2->cid_1)
    # DO NOT RUN
    if: "${{ inputs.NOT_EXIST_VAR }}"
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy [cid_2]' (deploy cid_2)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy ${{ env.MO_TEST_CID_2 }}"
          mo_ctl deploy ${{ env.MO_TEST_CID_2 }}
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. 'mo_ctl start' (start mo-service) and prepare some data
        run: |
          mo_ctl start
          echo "Sleeping 30 seconds for mo to be ready"
          sleep 30
          echo "Prepare some data"
          mo_ctl sql "create database test; use test; create table tb1(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"
      - name: Test_3. 'mo_ctl upgrade [cid_1]' (cid_2 -> cid_1)
        run: |
          if ! mo_ctl stop; then
              mo_ctl stop force
          fi
          echo "Sleep 5 seconds before upgrading"
          sleep 5
          mo_ctl upgrade ${{ env.MO_TEST_CID_1 }}
          if ! mo_ctl start; then
              echo "Start mo-service failed. This may be due to incompatible data format when upgrading from one cid to another. Ignoring... "
          fi
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          if ! mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"; then
              echo "Run query failed. This may be due to incompatible data format when upgrading from one cid to another. Ignoring... "
          fi

  Job_19:
    name: Job_19 - upgrade (stable->main)
    if: "${{ inputs.NOT_EXIST_VAR }}"
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy [current_stable_version]' (deploy current stable version)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy ${{ env.MO_STABLE_VERSION }}"
          mo_ctl deploy ${{ env.MO_STABLE_VERSION }}
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. 'mo_ctl start' (start mo-service) and prepare some data
        run: |
          mo_ctl start
          echo "Sleeping 30 seconds for mo to be ready"
          sleep 30
          echo "Prepare some data"
          mo_ctl sql "create database test; use test; create table tb1(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"
      - name: Test_3. 'mo_ctl upgrade latest' (current stable version -> latest commit id)
        run: |
          if ! mo_ctl stop; then
              mo_ctl stop force
          fi
          echo "Sleep 5 seconds before upgrading"
          sleep 5
          mo_ctl upgrade latest
          mo_ctl start
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          if ! mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"; then
              echo "Run query failed. This may be due to incompatible data format when upgrading from one cid to another"
          fi

  Job_20:
    name: Job_20 - upgrade (main->stable)
    # DO NOT RUN
    if: "${{ inputs.NOT_EXIST_VAR }}"
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy main' (deploy latest commit id)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy latest"
          mo_ctl deploy main
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. 'mo_ctl start' (start mo-service) and prepare some data
        run: |
          mo_ctl start
          echo "Sleeping 30 seconds for mo to be ready"
          sleep 30
          echo "Prepare some data"
          mo_ctl sql "create database test; use test; create table tb1(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"
      - name: Test_3. 'mo_ctl upgrade [current_stable_version]' (latest commit id -> current stable version)
        run: |
          if ! mo_ctl stop; then
              mo_ctl stop force
          fi
          echo "Sleep 5 seconds before upgrading"
          sleep 5
          mo_ctl upgrade ${{ env.MO_STABLE_VERSION }}
          if ! mo_ctl start; then
              echo "Start mo-service failed. This may be due to incompatible data format when upgrading from one cid to another. Ignoring... "
          fi
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          if ! mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"; then
              echo "Run query failed. This may be due to incompatible data format when upgrading from one cid to another. Ignoring... "
          fi

  Job_21:
    name: Job_21 - upgrade (stable->cid_1)
    # DO NOT RUN
    if: "${{ inputs.NOT_EXIST_VAR }}"
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy [current_stable_version]' (deploy current stable version)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy ${{ env.MO_STABLE_VERSION }}"
          mo_ctl deploy ${{ env.MO_STABLE_VERSION }}
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. 'mo_ctl start' (start mo-service) and prepare some data
        run: |
          mo_ctl start
          echo "Sleeping 30 seconds for mo to be ready"
          sleep 30
          echo "Prepare some data"
          mo_ctl sql "create database test; use test; create table tb1(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"
      - name: Test_3. 'mo_ctl upgrade [cid_1]' (current stable version -> cid_1)
        run: |
          if ! mo_ctl stop; then
              mo_ctl stop force
          fi
          echo "Sleep 5 seconds before upgrading"
          sleep 5
          mo_ctl upgrade ${{ env.MO_TEST_CID_1 }}
          mo_ctl start
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          if ! mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"; then
              echo "Run query failed. This may be due to incompatible data format when upgrading from one cid to another"
          fi

  Job_22:
    name: Job_22 - upgrade (cid_1->stable)
    # DO NOT RUN
    if: "${{ inputs.NOT_EXIST_VAR }}"
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy [cid_1]' (deploy cid_1)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy ${{ env.MO_TEST_CID_1 }}"
          mo_ctl deploy ${{ env.MO_TEST_CID_1 }}
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. 'mo_ctl start' (start mo-service) and prepare some data
        run: |
          mo_ctl start
          echo "Sleeping 30 seconds for mo to be ready"
          sleep 30
          echo "Prepare some data"
          mo_ctl sql "create database test; use test; create table tb1(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"
      - name: Test_3. 'mo_ctl upgrade [current_stable_version]' (cid_1 -> current stable version)
        run: |
          if ! mo_ctl stop; then
              mo_ctl stop force
          fi
          echo "Sleep 5 seconds before upgrading"
          sleep 5
          mo_ctl upgrade ${{ env.MO_STABLE_VERSION }}
          if ! mo_ctl start; then
              echo "Start mo-service failed. This may be due to incompatible data format when upgrading from one cid to another. Ignoring... "
          fi
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          if ! mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"; then
              echo "Run query failed. This may be due to incompatible data format when upgrading from one cid to another. Ignoring... "
          fi


  Job_23:
    name: Job_23 - upgrade (cid_1->latest)
    # DO NOT RUN
    if: "${{ inputs.NOT_EXIST_VAR }}"
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy [cid_1]' (deploy cid_1)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy ${{ env.MO_TEST_CID_1 }}"
          mo_ctl deploy ${{ env.MO_TEST_CID_1 }}
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. 'mo_ctl start' (start mo-service) and prepare some data
        run: |
          mo_ctl start
          echo "Sleeping 30 seconds for mo to be ready"
          sleep 30
          echo "Prepare some data"
          mo_ctl sql "create database test; use test; create table tb1(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"
      - name: Test_3. 'mo_ctl upgrade latest' (cid_1 -> latest)
        run: |
          if ! mo_ctl stop; then
              mo_ctl stop force
          fi
          sleep 5
          echo "Sleep 5 seconds before upgrading"
          mo_ctl upgrade latest
          mo_ctl start
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          if ! mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"; then
              echo "Run query failed. This may be due to incompatible data format when upgrading from one cid to another"
          fi  
  Job_24:
    name: Job_24 - upgrade (latest->cid_1)
    # DO NOT RUN
    if: "${{ inputs.NOT_EXIST_VAR }}"
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy main' (deploy latest)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy main"
          mo_ctl deploy main
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. 'mo_ctl start' (start mo-service) and prepare some data
        run: |
          mo_ctl start
          echo "Sleeping 30 seconds for mo to be ready"
          sleep 30
          echo "Prepare some data"
          mo_ctl sql "create database test; use test; create table tb1(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"
      - name: Test_3. 'mo_ctl upgrade [cid_1]' (latest -> cid_1)
        run: |
          if ! mo_ctl stop; then
              mo_ctl stop force
          fi
          echo "Sleep 5 seconds before upgrading"
          sleep 5
          mo_ctl upgrade ${{ env.MO_STABLE_VERSION }}
          if ! mo_ctl start; then
              echo "Start mo-service failed. This may be due to incompatible data format when upgrading from one cid to another. Ignoring... "
          fi
          echo "Sleep 30 seconds for mo to be ready"
          sleep 30
          if ! mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"; then
              echo "Run query failed. This may be due to incompatible data format when upgrading from one cid to another. Ignoring... "
          fi

  Job_25:
    name: Job_25 - deploy | uninstall (docker)
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
      - name: Install docker
        uses: docker-practice/actions-setup-docker@master
        timeout-minutes: 12
      - name: set confs
        run: |
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
          mo_ctl set_conf MO_DEPLOY_MODE="docker"
          mo_ctl set_conf MO_CONTAINER_DATA_HOST_PATH="${{ env.MO_PATH }}"
          mo_ctl set_conf MO_CONTAINER_LIMIT_CPU="1"
          mo_ctl set_conf MO_CONTAINER_AUTO_RESTART="yes"
          mo_ctl set_conf MO_CONTAINER_IMAGE="matrixorigin/matrixone:latest"
      - name: Test_1. 'mo_ctl precheck'
        run: |
          echo ""
          if ! mo_ctl status; then 
              echo "'mo_ctl status' fails as expected"
          fi
      - name: Test_2. 'mo_ctl status' (no mo-service running)
        run: |
          echo ""
          if ! mo_ctl status; then 
              echo "'mo_ctl status' fails as expected"
          fi
      - name: Test_3. 'mo_ctl deploy'
        run: |
          echo ""
          mo_ctl deploy
      - name: Test_4. 'mo_ctl status' (mo-service not running)
        run: |
          echo ""
          if ! mo_ctl status; then
              echo "'mo_ctl status' failed as expected"
          else
              exit 1
          fi
      - name: Test_5. 'mo_ctl start'
        run: |
          echo ""
          mo_ctl start && mo_ctl status
      - name: Test_6. 'mo_ctl status' (mo-service running)
        run: |
          echo ""
          mo_ctl status
          echo "'mo_ctl status' succeeds as expected"
      - name: Test_7. 'mo_ctl stop'
        run: |
          echo ""
          mo_ctl stop
          if ! mo_ctl status; then 
              echo "Stop as expected"
          fi
      - name: Test_8. 'mo_ctl restart'
        run: |
          echo ""
          mo_ctl restart && mo_ctl status
      - name: Test_9. 'mo_ctl stop force'
        run: |
          echo ""
          mo_ctl stop force
          if ! mo_ctl status; then 
              echo "Stop force as expected"
          fi
      - name: Test_10. 'mo_ctl restart force'
        run: |
          echo ""
          mo_ctl restart force && mo_ctl status
          sleep 30
      - name: Test_11. 'mo_ctl pprof'
        run: |
          echo ""
          mkdir -p ${{ env.MO_PPROF_PATH }}
          mo_ctl set_conf PPROF_OUT_PATH="${{ env.MO_PPROF_PATH }}"
          mo_ctl pprof
          mo_ctl pprof profile
          mo_ctl pprof profile 30
          mo_ctl pprof allocs
          mo_ctl pprof heap
      - name: Test_12. 'mo_ctl connect'
        run: |
          echo ""
          echo "show databases;" | mo_ctl connect
      - name: Test_13. 'mo_ctl sql'
        run: |
          echo ""
          mo_ctl sql 'select version();' 
      - name: Test_14. 'mo_ctl uninstall'
        run: |
          echo ""
          mo_ctl stop
          echo "yes" | mo_ctl uninstall
      - name: Test_15. 'Debug get all confs'
        run: |
          echo ""
          mo_ctl get_conf

  Job_26:
    name: Job_26 - auto_clean_logs
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Test_1. 'mo_ctl deploy main' (deploy latest)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy main"
          mo_ctl deploy main
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. 'mo_ctl start' (start mo-service) and prepare some data
        run: |
          mo_ctl start
          echo "Sleeping 30 seconds for mo to be ready"
          sleep 30
          echo "Prepare some data"
          mo_ctl sql "create database test; use test; create table tb1(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "create database test2; use test2; create table tb2(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"

          mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"
      - name: Test_3. Set confs
        run: |
          mo_ctl set_conf TOOL_LOG_LEVEL=D
          mo_ctl set_conf TOOL_LOG_PATH="${{ env.MO_CTL_TOOL_LOG_PATH }}"
          mo_ctl set_conf CLEAN_LOGS_DAYS_BEFORE="0"
          mo_ctl set_conf CLEAN_LOGS_TABLE_LIST="statement_info,rawlog,metric"
          mo_ctl set_conf CLEAN_LOGS_CRON_SCHEDULE="* * * * *"
      - name: Test_4. Clean logs manually and automatically
        run: |
          echo "----------------------------------"
          echo "Clean logs manually"
          mo_ctl clean_logs
          
          echo "----------------------------------"
          echo "Check auto clean logs"
          if ! mo_ctl auto_clean_logs; then
              echo "failed as expected"
          fi
          if ! mo_ctl auto_clean_logs status; then
              echo "failed as expected"
          fi

          echo "----------------------------------"
          echo "Enable auto clean logs"
          mo_ctl auto_clean_logs enable
          sleep 5
          echo "check log path"
          ls -lth ${{ env.MO_CTL_TOOL_LOG_PATH }}/auto_clean_logs/
          echo "check cron.d path and file"
          ls -lth /etc/cron.d/
          echo "content of file /etc/cron.d/mo_clean_logs"
          cat /etc/cron.d/mo_clean_logs

          echo "Disabling auto clean logs"
          mo_ctl auto_clean_logs disable
          if ! mo_ctl auto_clean_logs; then
              echo "auto_clean_logs status failed as expected"
          fi


  Job_27:
    name: Job_27 - backup | auto_backup (logical)
    runs-on: ubuntu-latest
    steps:
      - name: Initialize environment
        run: |
          echo "install mo_ctl with Internet(no proxy)"
          wget ${{ env.MO_CTL_GIT_URL }}/install.sh && bash +x ./install.sh
          mkdir -p ${{ env.MO_PATH }}
      - name: Download tools
        run: |
          tools=("mo_dump" "mo-tpch" "mo-backup")
          echo "Download and extract tools"
          mkdir -p ${{ env.MO_TOOL_PATH }}
          for tool in ${tools[*]}; do
              echo "Tool name: ${tool}"
              wget ${{ env.MO_TOOL_URL }}/${tool}.tar.gz -O ${{ env.MO_TOOL_PATH }}/${tool}.tar.gz
              tar xvf ${{ env.MO_TOOL_PATH }}/${tool}.tar.gz -C ${{ env.MO_TOOL_PATH }}/
          done
          
      - name: Test_1. 'mo_ctl deploy main' (deploy latest)
        run: |
          echo ""
          RUN_TAG="$(date "+%Y%m%d_%H%M%S")"
          mo_ctl set_conf MO_PATH="${{ env.MO_PATH }}/${RUN_TAG}"
          mo_ctl set_conf GOPROXY="${{ env.GO_PROXY }}"
          
          echo "MO_PATH: ${{ env.MO_PATH }}/${RUN_TAG}, test_cmd: mo_ctl deploy main"
          mo_ctl deploy main
          mo_ctl get_cid
          mo_ctl set_conf TOOL_LOG_LEVEL="D"
      - name: Test_2. 'mo_ctl start' (start mo-service) and prepare some data
        run: |
          mo_ctl start
          echo "Sleeping 30 seconds for mo to be ready"
          sleep 30
          echo "Prepare some data"
          mo_ctl sql "create database test; use test; create table tb1(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "create database test2; use test2; create table tb2(id int, name varchar(50)); insert into test.tb1 values (1, 'Andy'),(2,'Becky'),(3,'Cindy'),(4,'David'),(5,'Emma'),(6,'Francis')"
          mo_ctl sql "select count(*) from test.tb1; select * from test.tb1"
      - name: Test_3. Load tpch 1g data into mo
        run: |
          cd ${{ env.MO_TOOL_PATH }}/mo-tpch
          ./run.sh -s 1 -g
          sleep 2
          ./run.sh -s 1 -c
          ./run.sh -s 1 -l

          db_name="tpch_1g"

          echo "------------------------"
          echo "Show databases"
          mo_ctl sql "show databases;"
          echo "------------------------"
          echo "Show tables in database ${db_name}"
          mo_ctl sql "show tables in ${db_name};"
          echo "Count table rows"
          sql=""
          tables=`MYSQL_PWD=${{ env.MO_PW }} mysql -u${{ env.MO_USER }} -P${{ env.MO_PORT }} -h${{ env.MO_HOST }} -e "show tables in ${db_name};" -N -s`
          for table in ${tables}; do
              #echo "table: ${table}"
              sql="${sql} union select '${table}' as table_name, count(*) from ${db_name}.${table}"
          done
          len=${#sql}
          start_pos=6
          sql_2=${sql:${start_pos}:${len}}
          mo_ctl sql "${sql_2}"     
      - name: Test_4. Set confs (common)
        run: |
          mo_ctl set_conf TOOL_LOG_PATH="${{ env.MO_CTL_TOOL_LOG_PATH }}"
          mo_ctl set_conf BACKUP_DATA_PATH="${{ env.BACKUP_PATH }}"
          mo_ctl auto_backup disable
          mkdir -p ${BACKUP_PATH}
          RUN_MONTH=`date '+%Y%m'`
          echo "RUN_MONTH=${RUN_MONTH}" >>$GITHUB_ENV
      - name: Test_5. Manual backup (logical, insert)
        run: |
          echo "--------------------------------------------"
          mo_ctl set_conf BACKUP_TYPE="logical"
          mo_ctl set_conf BACKUP_MODUMP_PATH="${{ env.MO_TOOL_PATH }}/mo_dump/mo-dump"
          mo_ctl set_conf BACKUP_LOGICAL_DATA_TYPE="insert"
          mo_ctl set_conf BACKUP_LOGICAL_DB_LIST="all_no_sysdb"
          mo_ctl backup
          for dir in `ls ${{ env.BACKUP_PATH }}/${RUN_MONTH}/`; do
              echo "--------------------------------------------"
              echo "Listing backup dir: ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/"
              ls -lth ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/
          done

      - name: Test_6. Manual backup (logical, csv)
        run: |
          echo "--------------------------------------------"
          mo_ctl set_conf BACKUP_TYPE="logical"
          mo_ctl set_conf BACKUP_MODUMP_PATH="${{ env.MO_TOOL_PATH }}/mo_dump/mo-dump"
          mo_ctl set_conf BACKUP_LOGICAL_DATA_TYPE="csv"
          mo_ctl set_conf BACKUP_LOGICAL_DB_LIST="all_no_sysdb"
          mo_ctl backup
          for dir in `ls ${{ env.BACKUP_PATH }}/${RUN_MONTH}/`; do
              echo "--------------------------------------------"
              echo "Listing backup dir: ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/"
              ls -lth ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/
          done

      - name: Test_7. Manual backup (physical)
        run: |
          echo "--------------------------------------------"
          mo_ctl set_conf BACKUP_TYPE="physical"
          mo_ctl set_conf BACKUP_MOBR_PATH="${{ env.MO_TOOL_PATH }}/mo-backup/mo_br"
          mo_ctl set_conf BACKUP_PHYSICAL_TYPE="filesystem"
          mo_ctl backup
          for dir in `ls ${{ env.BACKUP_PATH }}/${RUN_MONTH}/`; do
              echo "--------------------------------------------"
              echo "Listing backup dir: ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/"
              ls -lth ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/
          done

      - name: Test_8. List backup files (manual)
        run: |
          echo "----------------------------------"
          echo "mo_ctl backup list (physical+logical)"
          mo_ctl backup list

          echo "----------------------------------"
          echo "mo_ctl backup list detail (physical only)"
          mo_ctl backup list detail


      - name: Test_9. Clean backup files (manual)
        run: |
          echo ""
          echo "----------------------------------"
          echo "Set confs for auto backup"
          mo_ctl set_conf BACKUP_CLEAN_DAYS_BEFORE=0

          echo "----------------------------------"
          echo "mo_ctl clean_backup"
          mo_ctl clean_backup

          sleep 5

          dir_count=`ls ${{ env.BACKUP_PATH }}/${RUN_MONTH} | wc -l`

          if [[ ${dir_count} -ne 0 ]]; then
              echo "dir_count: ${dir_count} is not 0, seems like clean_backup has failed"
              for dir in `ls ${{ env.BACKUP_PATH }}/${RUN_MONTH}/`; do
                  echo "--------------------------------------------"
                  echo "Listing backup dir: ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/"
                  ls -lth ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/
              done
              exit 1
          fi

      - name: Test_10. Auto backup (physical)
        run: |

          echo ""
          echo "----------------------------------"
          echo "Set confs for auto backup"
          mo_ctl set_conf BACKUP_CRON_SCHEDULE="* * * * *"
          mo_ctl set_conf BACKUP_CLEAN_DAYS_BEFORE="0"
          mo_ctl set_conf BACKUP_CLEAN_CRON_SCHEDULE="*/8 * * * *"


          echo ""
          echo "----------------------------------"
          echo "Disable auto backup"
          mo_ctl auto_backup disable

          echo ""
          echo "----------------------------------"
          echo "Check auto backup status"
          if ! mo_ctl auto_backup; then
              echo "auto_backup status failed as expected"
          fi

          if ! mo_ctl auto_backup status; then
              echo "auto_backup status failed as expected"
          fi

          echo ""
          echo "----------------------------------"
          echo "Enable auto backup"
          mo_ctl set_conf BACKUP_CLEAN_DAYS_BEFORE=0

          mo_ctl auto_backup enable

          echo ""
          echo "----------------------------------"
          echo "Check auto backup status"
          mo_ctl auto_backup

          echo "Wait for 150s for next auto backup to perform"
          sleep 150

          for dir in `ls ${{ env.BACKUP_PATH }}/${RUN_MONTH}/`; do
              echo "--------------------------------------------"
              echo "Listing backup dir: ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/"
              ls -lth ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/
          done

          if [[ ! -d ${{ env.BACKUP_PATH }}/${RUN_MONTH}/ ]]; then
              echo "Directory ${{ env.BACKUP_PATH }}/${RUN_MONTH}/ does not exist, seems like auto_backup has failed"
              exit 1
          else
              dir_count=`ls ${{ env.BACKUP_PATH }}/${RUN_MONTH}/ | wc -l`
              if [[ ${dir_count} -eq 0 ]]; then
                  echo "dir_count: ${dir_count} is 0, seems like auto_backup has failed"
                  exit 1
              fi 

          fi

          # disable auto backup since it may affect below tests
          mo_ctl auto_backup disable

          if mo_ctl auto_backup || mo_ctl auto_backup status; then
              echo "Auto backup is not disabled as expected, exiting"
              exit 1
          fi

      - name: Test_11. Clean backup files (Auto)
        run: |

          echo "----------------------------------"
          echo "Set confs for auto backup"
          # set auto backup to a specific timing
          mo_ctl set_conf BACKUP_CRON_SCHEDULE="30 6 * * *"
          mo_ctl set_conf BACKUP_CLEAN_DAYS_BEFORE="0"
          # set auto clean backup to once every 2 minute
          mo_ctl set_conf BACKUP_CLEAN_CRON_SCHEDULE="*/2 * * * *"

         
          echo "----------------------------------"
          echo "Enable auto backup and auto clean backup"
          mo_ctl auto_backup enable

          echo "----------------------------------"
          echo "Check auto backup and auto clean backup status"
          mo_ctl auto_backup status         

          echo "----------------------------------"
          echo "Backup manuallyu"
          mo_ctl backup

          echo "----------------------------------"
          echo "Show backup path"
          ls -lth ${{ env.BACKUP_PATH }}
          for dir in `ls ${{ env.BACKUP_PATH }}`; do
              echo "----------------------------------"
              echo "Listing files in ${{ env.BACKUP_PATH }}/${dir}/"
              ls -lth ${{ env.BACKUP_PATH }}/${dir}/
          done

          echo "----------------------------------"
          echo "Clean backup automatically"
          mo_ctl auto_backup
          echo "Sleep 130s"
          sleep 130

          echo "Show backup path: ls -lth ${{ env.BACKUP_PATH }}/${RUN_MONTH}/"
          ls -lth ${{ env.BACKUP_PATH }}/${RUN_MONTH}/

          for dir in `ls ${{ env.BACKUP_PATH }}/${RUN_MONTH}/`; do
              echo "----------------------------------"
              echo "Listing files in ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/"
              ls -lth ${{ env.BACKUP_PATH }}/${RUN_MONTH}/${dir}/
          done

          dir_count=`ls ${{ env.BACKUP_PATH }}/${RUN_MONTH}/ | wc -l`
          if [[ ${dir_count} -ne 0 ]]; then
              echo "dir_count: ${dir_count} is not 0, seems like auto_clean_old_backup has failed"
              exit 1
          fi 

          # for debuging
          #echo "----------------------------------"
          #echo "check log path ${MO_CTL_TOOL_LOG_PATH}/auto_backup/"
          #ls -lth ${MO_CTL_TOOL_LOG_PATH}/auto_backup/

          #echo "----------------------------------"
          #echo "check auto_backup logs: cat ${MO_CTL_TOOL_LOG_PATH}/auto_backup/*"
          #cat ${MO_CTL_TOOL_LOG_PATH}/auto_backup/*

          #echo "----------------------------------"
          #echo "check log path ${MO_CTL_TOOL_LOG_PATH}/auto_clean_old_backup/"
          #ls -lth ${MO_CTL_TOOL_LOG_PATH}/auto_clean_old_backup/

          #echo "----------------------------------"
          #echo "show auto_clean_old_backup logs: cat ${MO_CTL_TOOL_LOG_PATH}/auto_clean_old_backup/*"
          #cat ${MO_CTL_TOOL_LOG_PATH}/auto_clean_old_backup/*
          
          #echo "----------------------------------"
          #echo "check cron.d path and file"
          #ls -lth /etc/cron.d/
          
          #echo "----------------------------------"
          #echo "content of file /etc/cron.d/mo_backup"
          #cat /etc/cron.d/mo_backup
          
          #echo "----------------------------------"
          #echo "content of file /etc/cron.d/mo_clean_old_backup"
          #cat /etc/cron.d/mo_clean_old_backup
            
          #echo "----------------------------------"
          #echo "Disabling auto_backup"
          #mo_ctl auto_backup disable
          #if ! mo_ctl auto_backup; then
          #    echo "auto_backup status failed as expected"
          #fi