fix tabledump and analyze parameter

handler/analyzer/analyze_parameter.py

@@ -162,7 +162,7 @@ def analyze_parameter_default(self):
                 report_default_tb.add_row([row[1], row[2], row[3], row[4], tenant_id, row[6], row[11], row[7]])
             fp.write(report_default_tb.get_string() + "\n")
             self.stdio.print(report_default_tb.get_string())
-            self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL)
+            self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'")
         else:
             if self.parameter_file_name is None:
                 self.stdio.error("the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values")

handler/gather/gather_parameters.py

@@ -121,7 +121,7 @@ def get_parameters_info(self):
                         writer.writerow(tmp_row)
                     else:
                         writer.writerow(row)
-            self.stdio.print("Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.parameter_file_name) + Style.RESET_ALL + "'")
+            self.stdio.print("Gather parameters finished. For more details, please run cmd " + Fore.YELLOW + "cat '{0}".format(self.parameter_file_name) + Style.RESET_ALL + "'")
         else:
             self.stdio.warn("Failed to retrieve the database version. Please check if the database connection is normal.")
 

handler/gather/gather_tabledump.py

@@ -62,8 +62,8 @@ def init(self):
             self.table = Util.get_option(options, 'table')
             user = Util.get_option(options, 'user')
             password = Util.get_option(options, 'password')
-            if not (self.database and self.database and user and password):
-                self.stdio.error("option --database/--table/--user/--password not found, please provide")
+            if not (self.database and self.table and user):
+                self.stdio.error("option --database/--table/--user not found, please provide")
                 return False
             store_dir_option = Util.get_option(options, 'store_dir')
             if store_dir_option is not None and store_dir_option != './':
@@ -100,27 +100,31 @@ def handle(self):
         if not self.init():
             self.stdio.error('init failed')
             return False
-        self.execute()
-        if not self.is_innner:
+        excute_status = self.execute()
+        if not self.is_innner and excute_status:
             self.__print_result()
 
     def execute(self):
         try:
             self.version = get_observer_version(self.context)
-            self.__get_table_schema()
-            if self.version == "4.0.0.0" or StringUtils.compare_versions_greater(self.version, "4.0.0.0"):
-                self.__get_table_info()
-            else:
-                self.__get_table_info_v3()
+            if self.__get_table_schema():
+                if self.version == "4.0.0.0" or StringUtils.compare_versions_greater(self.version, "4.0.0.0"):
+                    return self.__get_table_info()
+                else:
+                    return self.__get_table_info_v3()
         except Exception as e:
-            self.stdio.error("report sql result to file: {0} failed, error: {1}".format(self.file_name, e))
+            self.stdio.error("report sql result failed, error: {0}".format(e))
 
     def __get_table_schema(self):
-        sql = "show create table " + self.database + "." + self.table
-        columns, result = self.tenant_connector.execute_sql_return_columns_and_data(sql)
-        if result is None or len(result) == 0:
-            self.stdio.verbose("excute sql: {0},  result is None".format(sql))
-        self.__report(sql, columns, result)
+        try:
+            sql = "show create table " + self.database + "." + self.table
+            columns, result = self.tenant_connector.execute_sql_return_columns_and_data(sql)
+            if result is None or len(result) == 0:
+                self.stdio.verbose("excute sql: {0},  result is None".format(sql))
+            self.__report(sql, columns, result)
+            return True
+        except Exception as e:
+            self.stdio.error("show create table error {0}".format(e))
 
     def __get_table_info(self):
         try:
@@ -172,6 +176,7 @@ def __get_table_info(self):
                 return
             self.stdio.print("data size {0}".format(result))
             self.__report(query_data, columns, result)
+            return True
 
         except Exception as e:
             self.stdio.error("getTableInfo execute Exception: {0}".format(e).strip())
@@ -203,6 +208,7 @@ def __get_table_info_v3(self):
                 return
             self.stdio.print("table count {0}".format(result))
             self.__report(query_count, columns, result)
+            return True
 
         except Exception as e:
             self.stdio.error("getTableInfo execute Exception: {0}".format(e).strip())
@@ -215,7 +221,7 @@ def __report(self, sql, column_names, data):
                 f.write('\n\n' + 'obclient > ' + sql + '\n')
                 f.write(formatted_table)
         except Exception as e:
-            self.stdio.error("report sql result to file: {0} failed, error: ".format(self.file_name))
+            self.stdio.error("report sql result to file: {0} failed, error:{1} ".format(self.file_name, e))
 
     def __extract_string(self, s):
         if '@' in s:

handler/gather/tasks/observer/clog_disk_full.yaml

@@ -105,15 +105,6 @@ task:
       - type: sql
         sql: "select * from oceanbase.DBA_OB_SERVER_EVENT_HISTORY where event like '%migrat%' and name6 like '%fail%' and value6=1;"
         global: true
-      - type: sql
-        sql: "select b.zone, a.svr_ip,a.svr_port,b.with_rootserver, round(a.total_size/1024/1024/1024,0) as total_GB,round(a.free_size/1024/1024/1024,0) as free_GB, round((a.total_size-a.free_size)/1024/1024/1024,0) as used_GB,(a.total_size-a.free_size)/a.total_size as used_percent from oceanbase.__all_virtual_disk_stat a, oceanbase.DBA_OB_SERVERS b where a.svr_ip=b.svr_ip and a.svr_port=b.svr_port order by b.zone,used_percent desc;"
-        global: true
-      - type: sql
-        sql: "select tenant_name, svr_ip,  memstore_limit /(1024 * 1024 * 1024) as memstore_limit_GB,  freeze_trigger /(1024 * 1024 * 1024) as freeze_trigger_GB,  memstore_used /(1024 * 1024 * 1024) as memstore_used_GB,  concat((memstore_used * 100 / memstore_limit), '%') as memstore_used_percent,  active_span /(1024 * 1024 * 1024) as active_span_GB,  freeze_cnt   from oceanbase.GV$OB_MEMSTORE memstore_info  inner join oceanbase.DBA_OB_TENANTS tenant on  memstore_info.tenant_id = tenant.tenant_id  ORDER BY tenant.tenant_name,svr_ip;"
-        global: true
-      - type: sql # 查看每台机器上的tablet分布
-        sql: "select svr_ip, svr_port, ls_id, count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS group by svr_ip, svr_port, ls_id;"
-        global: true
       - type: ssh
         ssh: "df -h"
         global: false

handler/gather/tasks/observer/compaction.yaml

@@ -150,15 +150,6 @@ task:
       - type: sql
         sql: "select t.tenant_name, t1.database_name, round(sum(t2.data_size)/1024/1024/1024,2) as data_size_gb, round(sum(t2.required_size)/1024/1024/1024,2) as required_size_gb from oceanbase.dba_ob_tenants t, oceanbase.cdb_ob_table_locations t1, oceanbase.cdb_ob_tablet_replicas t2 where t.tenant_id=t1.tenant_id and t1.svr_ip=t2.svr_ip and t1.tenant_id=t2.tenant_id and t1.ls_id=t2.ls_id and t1.tablet_id=t2.tablet_id and t1.role='leader' group by t.tenant_name, t1.database_name order by data_size_gb desc;"
         global: true
-      - type: sql
-        sql: "select b.zone, a.svr_ip,a.svr_port,b.with_rootserver, round(a.total_size/1024/1024/1024,0) as total_GB,round(a.free_size/1024/1024/1024,0) as free_GB, round((a.total_size-a.free_size)/1024/1024/1024,0) as used_GB,(a.total_size-a.free_size)/a.total_size as used_percent from oceanbase.__all_virtual_disk_stat a, oceanbase.DBA_OB_SERVERS b where a.svr_ip=b.svr_ip and a.svr_port=b.svr_port order by b.zone,used_percent desc;"
-        global: true
-      - type: sql
-        sql: "select tenant_name, svr_ip,  memstore_limit /(1024 * 1024 * 1024) as memstore_limit_GB,  freeze_trigger /(1024 * 1024 * 1024) as freeze_trigger_GB,  memstore_used /(1024 * 1024 * 1024) as memstore_used_GB,  concat((memstore_used * 100 / memstore_limit), '%') as memstore_used_percent,  active_span /(1024 * 1024 * 1024) as active_span_GB,  freeze_cnt   from oceanbase.GV$OB_MEMSTORE memstore_info  inner join oceanbase.DBA_OB_TENANTS tenant on  memstore_info.tenant_id = tenant.tenant_id  ORDER BY tenant.tenant_name,svr_ip;"
-        global: true
-      - type: sql # 查看每台机器上的tablet分布
-        sql: "select svr_ip, svr_port, ls_id, count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS group by svr_ip, svr_port, ls_id;"
-        global: true
       - type: log
         global: false
         grep: ""

handler/gather/tasks/observer/delay_of_primary_and_backup.yaml

@@ -156,12 +156,6 @@ task:
       - type: sql
         sql: "SELECT LS_ID, SCN_TO_TIMESTAMP(END_SCN) FROM oceanbase.GV$OB_LOG_STAT WHERE ROLE = 'LEADER';"
         global: true
-      - type: sql
-        sql: "select b.zone, a.svr_ip,a.svr_port,b.with_rootserver, round(a.total_size/1024/1024/1024,0) as total_GB,round(a.free_size/1024/1024/1024,0) as free_GB, round((a.total_size-a.free_size)/1024/1024/1024,0) as used_GB,(a.total_size-a.free_size)/a.total_size as used_percent from oceanbase.__all_virtual_disk_stat a, oceanbase.DBA_OB_SERVERS b where a.svr_ip=b.svr_ip and a.svr_port=b.svr_port order by b.zone,used_percent desc;"
-        global: true
-      - type: sql # 查看每台机器上的tablet分布
-        sql: "select svr_ip, svr_port, ls_id, count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS group by svr_ip, svr_port, ls_id;"
-        global: true
       - type: log
         global: false
         grep: ""

handler/gather/tasks/observer/log_archive.yaml

@@ -120,9 +120,6 @@ task:
       - type: sql
         sql: "SELECT * FROM oceanbase.CDB_OB_ARCHIVELOG_PIECE_FILES limit 20"
         global: true
-      - type: sql # 查看每台机器上的tablet分布
-        sql: "select svr_ip, svr_port, ls_id, count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS group by svr_ip, svr_port, ls_id;"
-        global: true
       - type: log
         global: false
         grep: ""

handler/gather/tasks/observer/long_transaction.yaml

@@ -84,9 +84,6 @@ task:
       - type: sql
         sql: "SELECT count(1) FROM oceanbase.GV$OB_TRANSACTION_PARTICIPANTS WHERE CTX_CREATE_TIME < date_sub(now(), INTERVAL 600 SECOND) AND STATE = 'INIT';"
         global: true
-      - type: sql
-        sql: "select tenant_name, svr_ip,  memstore_limit /(1024 * 1024 * 1024) as memstore_limit_GB,  freeze_trigger /(1024 * 1024 * 1024) as freeze_trigger_GB,  memstore_used /(1024 * 1024 * 1024) as memstore_used_GB,  concat((memstore_used * 100 / memstore_limit), '%') as memstore_used_percent,  active_span /(1024 * 1024 * 1024) as active_span_GB,  freeze_cnt   from oceanbase.GV$OB_MEMSTORE memstore_info  inner join oceanbase.DBA_OB_TENANTS tenant on  memstore_info.tenant_id = tenant.tenant_id  ORDER BY tenant.tenant_name,svr_ip;"
-        global: true
       - type: log
         grep: ""
         global: false

handler/gather/tasks/observer/memory.yaml

@@ -94,10 +94,7 @@ task:
         sql: "show parameters like '%syslog_io_bandwidth_limit%';"
         global: true
       - type: sql
-        sql: "select tenant_name, svr_ip,  memstore_limit /(1024 * 1024 * 1024) as memstore_limit_GB,  freeze_trigger /(1024 * 1024 * 1024) as freeze_trigger_GB,  memstore_used /(1024 * 1024 * 1024) as memstore_used_GB,  concat((memstore_used * 100 / memstore_limit), '%') as memstore_used_percent,  active_span /(1024 * 1024 * 1024) as active_span_GB,  freeze_cnt   from oceanbase.GV$OB_MEMSTORE memstore_info  inner join oceanbase.DBA_OB_TENANTS tenant on  memstore_info.tenant_id = tenant.tenant_id  ORDER BY tenant.tenant_name,svr_ip;"
-        global: true
-      - type: sql # 大于1g的内存模块
-        sql: "SELECT CASE WHEN tenant_name IS NULL THEN TENANT_ID ELSE tenant_name END tenant_name, host,ctx_name, mod_name, hold, used, count FROM ( SELECT tenant_name,tenant_id,HOST,ctx_name,mod_name,hold,used,COUNT, ROW_NUMBER () OVER ( PARTITION BY tenant_name, HOST ORDER BY hold desc) rnum FROM   (SELECT  b.tenant_name, a.tenant_id, concat(a.svr_ip, ':', a.svr_port) HOST,  a.ctx_name, a.mod_name,  round(a.hold / 1024 / 1024 / 1024) hold, round(a.used / 1024 / 1024 / 1024) used, a.COUNT FROM  oceanbase.__all_virtual_memory_info a  LEFT JOIN oceanbase.__all_tenant b ON a.TENANT_ID = b.TENANT_ID  WHERE a.hold > 1024 * 1024 * 1024 ));"
+        sql: "select * from oceanbase.GV$OB_MEMSTORE limit 20"
         global: true
       - type: ssh # 可看到租户的规格、线程、队列及请求统计等信息，且这条日志每个租户每 30s 打印一次
         ssh: "grep 'dump tenant info.tenant=' ${observer_data_dir}/log/observer.log | sed 's/,/,/g'"

handler/gather/tasks/observer/recovery.yaml

@@ -89,9 +89,6 @@ task:
       - type: sql
         sql: "SELECT * FROM oceanbase.CDB_OB_RESTORE_HISTORY limit 20;"
         global: true
-      - type: sql # 查看每台机器上的tablet分布
-        sql: "select svr_ip, svr_port, ls_id, count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS group by svr_ip, svr_port, ls_id;"
-        global: true
       - type: log
         grep: ""
         global: false

handler/gather/tasks/observer/rootservice_switch.yaml

@@ -132,9 +132,6 @@ task:
       - type: sql
         sql: "SELECT TENANT_NAME, TENANT_ID, TENANT_ROLE, STATUS, SWITCHOVER_STATUS FROM oceanbase.DBA_OB_TENANTS"
         global: true
-      - type: sql # 大于1g的内存模块
-        sql: "SELECT CASE WHEN tenant_name IS NULL THEN TENANT_ID ELSE tenant_name END tenant_name, host,ctx_name, mod_name, hold, used, count FROM ( SELECT tenant_name,tenant_id,HOST,ctx_name,mod_name,hold,used,COUNT, ROW_NUMBER () OVER ( PARTITION BY tenant_name, HOST ORDER BY hold desc) rnum FROM   (SELECT  b.tenant_name, a.tenant_id, concat(a.svr_ip, ':', a.svr_port) HOST,  a.ctx_name, a.mod_name,  round(a.hold / 1024 / 1024 / 1024) hold, round(a.used / 1024 / 1024 / 1024) used, a.COUNT FROM  oceanbase.__all_virtual_memory_info a  LEFT JOIN oceanbase.__all_tenant b ON a.TENANT_ID = b.TENANT_ID  WHERE a.hold > 1024 * 1024 * 1024 ));"
-        global: true
       - type: log
         grep: ""
         global: false

handler/gather/tasks/observer/suspend_transaction.yaml

@@ -81,9 +81,6 @@ task:
       - type: sql
         sql: "SELECT count(1) FROM oceanbase.GV$OB_TRANSACTION_PARTICIPANTS WHERE CTX_CREATE_TIME < date_sub(now(), INTERVAL 600 SECOND) AND (STATE = 'PREPARE' OR STATE = 'REDO COMPLETE' OR STATE ='PRECOMMIT');"
         global: true
-      - type: sql
-        sql: "select tenant_name, svr_ip,  memstore_limit /(1024 * 1024 * 1024) as memstore_limit_GB,  freeze_trigger /(1024 * 1024 * 1024) as freeze_trigger_GB,  memstore_used /(1024 * 1024 * 1024) as memstore_used_GB,  concat((memstore_used * 100 / memstore_limit), '%') as memstore_used_percent,  active_span /(1024 * 1024 * 1024) as active_span_GB,  freeze_cnt   from oceanbase.GV$OB_MEMSTORE memstore_info  inner join oceanbase.DBA_OB_TENANTS tenant on  memstore_info.tenant_id = tenant.tenant_id  ORDER BY tenant.tenant_name,svr_ip;"
-        global: true
       - type: log
         grep: ""
         global: false

handler/gather/tasks/observer/unit_data_imbalance.yaml

@@ -154,10 +154,7 @@ task:
         sql: "select t.tenant_name, t1.database_name, round(sum(t2.data_size)/1024/1024/1024,2) as data_size_gb, round(sum(t2.required_size)/1024/1024/1024,2) as required_size_gb from oceanbase.dba_ob_tenants t, oceanbase.cdb_ob_table_locations t1, oceanbase.cdb_ob_tablet_replicas t2 where t.tenant_id=t1.tenant_id and t1.svr_ip=t2.svr_ip and t1.tenant_id=t2.tenant_id and t1.ls_id=t2.ls_id and t1.tablet_id=t2.tablet_id and t1.role='leader' group by t.tenant_name, t1.database_name order by data_size_gb desc;"
         global: true
       - type: sql
-        sql: "select b.zone, a.svr_ip,a.svr_port,b.with_rootserver, round(a.total_size/1024/1024/1024,0) as total_GB,round(a.free_size/1024/1024/1024,0) as free_GB, round((a.total_size-a.free_size)/1024/1024/1024,0) as used_GB,(a.total_size-a.free_size)/a.total_size as used_percent from oceanbase.__all_virtual_disk_stat a, oceanbase.DBA_OB_SERVERS b where a.svr_ip=b.svr_ip and a.svr_port=b.svr_port order by b.zone,used_percent desc;"
-        global: true
-      - type: sql # 查看每台机器上的tablet分布
-        sql: "select svr_ip, svr_port, ls_id, count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS group by svr_ip, svr_port, ls_id;"
+        sql: "select svr_ip,total_size / 1024 / 1024 / 1024 total_G,free_size / 1024 / 1024 / 1024 free_G,(total_size - free_size) / 1024 / 1024 / 1024 used_G,(total_size - free_size) / total_size used_percentage FROM oceanbase.__all_virtual_disk_stat;  "
         global: true
       - type: log
         grep: ""

handler/gather/tasks/observer/unknown.yaml

@@ -69,15 +69,6 @@ task:
       - type: sql
         sql: "SELECT a.TENANT_NAME,a.TENANT_ID,b.SVR_IP FROM oceanbase.DBA_OB_TENANTS a, oceanbase.GV$OB_UNITS b WHERE a.TENANT_ID=b.TENANT_ID;"
         global: true
-      - type: sql
-        sql: "select b.zone, a.svr_ip,a.svr_port,b.with_rootserver, round(a.total_size/1024/1024/1024,0) as total_GB,round(a.free_size/1024/1024/1024,0) as free_GB, round((a.total_size-a.free_size)/1024/1024/1024,0) as used_GB,(a.total_size-a.free_size)/a.total_size as used_percent from oceanbase.__all_virtual_disk_stat a, oceanbase.DBA_OB_SERVERS b where a.svr_ip=b.svr_ip and a.svr_port=b.svr_port order by b.zone,used_percent desc;"
-        global: true
-      - type: sql # 大于1g的内存模块
-        sql: "SELECT CASE WHEN tenant_name IS NULL THEN TENANT_ID ELSE tenant_name END tenant_name, host,ctx_name, mod_name, hold, used, count FROM ( SELECT tenant_name,tenant_id,HOST,ctx_name,mod_name,hold,used,COUNT, ROW_NUMBER () OVER ( PARTITION BY tenant_name, HOST ORDER BY hold desc) rnum FROM   (SELECT  b.tenant_name, a.tenant_id, concat(a.svr_ip, ':', a.svr_port) HOST,  a.ctx_name, a.mod_name,  round(a.hold / 1024 / 1024 / 1024) hold, round(a.used / 1024 / 1024 / 1024) used, a.COUNT FROM  oceanbase.__all_virtual_memory_info a  LEFT JOIN oceanbase.__all_tenant b ON a.TENANT_ID = b.TENANT_ID  WHERE a.hold > 1024 * 1024 * 1024 ));"
-        global: true
-      - type: sql # 查看每台机器上的tablet分布
-        sql: "select svr_ip, svr_port, ls_id, count(*) from oceanbase.CDB_OB_TABLE_LOCATIONS group by svr_ip, svr_port, ls_id;"
-        global: true
       - type: sql
         sql: "show parameters like '%syslog_level%';"
         global: true