diff --git a/handler/gather/step/base.py b/handler/gather/step/base.py index ee1bb9c2..5d11564c 100644 --- a/handler/gather/step/base.py +++ b/handler/gather/step/base.py @@ -63,7 +63,7 @@ def execute(self): handler = SshHandler(self.context, self.step, self.node, self.report_path, self.task_variable_dict) handler.execute() elif self.step["type"] == "sql" and (skip_type != "sql"): - handler = StepSQLHandler(self.context, self.step, self.cluster, self.report_path, self.task_variable_dict) + handler = StepSQLHandler(self.context, self.step, self.cluster, self.report_path, self.task_variable_dict, self.env) handler.execute() elif self.step["type"] == "log" and (skip_type != "ssh"): if self.node.get("host_type") and self.node.get("host_type") == "OBSERVER": diff --git a/handler/gather/step/sql.py b/handler/gather/step/sql.py index 2691daf9..8f6e8050 100644 --- a/handler/gather/step/sql.py +++ b/handler/gather/step/sql.py @@ -16,6 +16,7 @@ @desc: """ import os +import re from stdio import SafeStdio from common.ob_connector import OBConnector from tabulate import tabulate @@ -23,7 +24,7 @@ class StepSQLHandler(SafeStdio): - def __init__(self, context, step, ob_cluster, report_path, task_variable_dict): + def __init__(self, context, step, ob_cluster, report_path, task_variable_dict, env): self.context = context self.stdio = context.stdio try: @@ -32,6 +33,7 @@ def __init__(self, context, step, ob_cluster, report_path, task_variable_dict): self.tenant_mode = None self.sys_database = None self.database = None + self.env = env self.ob_connector = OBConnector(ip=ob_cluster.get("db_host"), port=ob_cluster.get("db_port"), username=ob_cluster.get("tenant_sys").get("user"), password=ob_cluster.get("tenant_sys").get("password"), stdio=self.stdio, timeout=10000) except Exception as e: self.stdio.error("StepSQLHandler init fail. Please check the OBCLUSTER conf. OBCLUSTER: {0} Exception : {1} .".format(ob_cluster, e)) @@ -49,6 +51,13 @@ def execute(self): self.stdio.error("StepSQLHandler execute sql is not set") return sql = StringUtils.build_sql_on_expr_by_dict(self.step["sql"], self.task_variable_dict) + params = StringUtils.extract_parameters(sql) + for param in params: + values = self.env.get(param) + if values is None or len(values) == 0: + self.stdio.error("the values of param %s is None", param) + return + sql = StringUtils.replace_parameters(sql, self.env) self.stdio.verbose("StepSQLHandler execute: {0}".format(sql)) columns, data = self.ob_connector.execute_sql_return_columns_and_data(sql) if data is None or len(data) == 0: @@ -57,6 +66,23 @@ def execute(self): except Exception as e: self.stdio.error("StepSQLHandler execute Exception: {0}".format(e).strip()) + @staticmethod + def extract_parameters(query_template): + pattern = re.compile(r'\$\{(\w+)\}') + parameters = pattern.findall(query_template) + return parameters + + @staticmethod + def replace_parameters(query_template, params): + pattern = re.compile(r'\$\{(\w+)\}') + + def replacer(match): + key = match.group(1) + return str(params.get(key, match.group(0))) + + query = pattern.sub(replacer, query_template) + return query + def update_step_variable_dict(self): return self.task_variable_dict diff --git a/handler/gather/tasks/observer/backup.yaml b/handler/gather/tasks/observer/backup.yaml index 6e7cc91f..63112953 100644 --- a/handler/gather/tasks/observer/backup.yaml +++ b/handler/gather/tasks/observer/backup.yaml @@ -1,6 +1,6 @@ info_en: "[backup problem]" info_cn: "[数据备份问题]" -command: obdiag gather scene run --scene=observer.backup +command: obdiag gather scene run --scene=observer.backup --env "{tenant_id=xxx}" task: - version: "[2.0.0.0, 4.0.0.0]" steps: @@ -53,7 +53,7 @@ task: sql: "select * from oceanbase.__all_rootservice_event_history where gmt_create > ${from_time} and gmt_create < ${to_time} order by gmt_create desc;" global: true - type: sql - sql: "select b.* from oceanbase.__all_virtual_pg_backup_log_archive_status a,oceanbase.__all_virtual_pg_log_archive_stat b where a.table_id=b.table_id and a.partition_id=b.partition_id order by log_archive_cur_ts limit 5;" + sql: "select b.* from oceanbase.__all_virtual_pg_backup_log_archive_status a,oceanbase.__all_virtual_pg_log_archive_stat b where a.table_id=b.table_id and a.partition_id=b.partition_id order by log_archive_cur_ts limit 100;" global: true - type: log global: false @@ -103,22 +103,22 @@ task: sql: "show parameters like '%ha_low_thread_score%';" global: true - type: sql - sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_PARAMETER" + sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_PARAMETER where tenant_id = ${tenant_id}" global: true - type: sql - sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_JOBS limit 20;" + sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_JOBS where tenant_id = ${tenant_id};" global: true - type: sql sql: "SELECT * FROM oceanbase.DBA_OB_ROOTSERVICE_EVENT_HISTORY WHERE module='backup_data' AND event ='start_backup_data';" global: true - type: sql - sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_TASKS limit 20;" + sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_TASKS where tenant_id = ${tenant_id};" global: true - type: sql - sql: "SELECT * FROM oceanbase.__all_virtual_backup_schedule_task limit 20" + sql: "SELECT * FROM oceanbase.__all_virtual_backup_schedule_task where tenant_id = ${tenant_id}" global: true - type: sql - sql: "SELECT * from oceanbase.CDB_OB_BACKUP_JOB_HISTORY where STATUS = 'FAILED' limit 20;" + sql: "SELECT * from oceanbase.CDB_OB_BACKUP_JOB_HISTORY where STATUS = 'FAILED' and tenant_id = ${tenant_id};" global: true - type: log global: false diff --git a/handler/gather/tasks/observer/backup_clean.yaml b/handler/gather/tasks/observer/backup_clean.yaml index 699ced1d..bd89ecc4 100644 --- a/handler/gather/tasks/observer/backup_clean.yaml +++ b/handler/gather/tasks/observer/backup_clean.yaml @@ -1,6 +1,6 @@ info_en: "[backup clean]" info_cn: "[备份清理问题]" -command: obdiag gather scene run --scene=observer.backup_clean +command: obdiag gather scene run --scene=observer.backup_clean --env "{tenant_id=xxx}" task: - version: "[2.0.0.0, 4.0.0.0]" steps: @@ -50,7 +50,7 @@ task: sql: "select * from oceanbase.__all_virtual_sys_task_status where comment like '%backup%';" global: true - type: sql - sql: "select * from oceanbase.CDB_OB_BACKUP_SET_DETAILS order by START_TIME asc limit 1;" + sql: "select * from oceanbase.CDB_OB_BACKUP_SET_DETAILS order by START_TIME desc" global: true - type: sql sql: "select * from oceanbase.__all_backup_task_clean_history where gmt_create > ${from_time} and gmt_create < ${to_time} order by gmt_create desc;" @@ -106,34 +106,34 @@ task: sql: "show parameters like '%backup%';" global: true - type: sql - sql: "select * from oceanbase.CDB_OB_BACKUP_JOB_HISTORY" + sql: "select * from oceanbase.CDB_OB_BACKUP_JOB_HISTORY where tenant_id=${tenant_id}" global: true - type: sql sql: "SELECT * FROM oceanbase.DBA_OB_ROOTSERVICE_EVENT_HISTORY WHERE module='backup_data' AND event ='start_backup_data';" global: true - type: sql - sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_TASKS limit 20;" + sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_TASKS where tenant_id=${tenant_id};" global: true - type: sql - sql: "SELECT * FROM oceanbase.__all_virtual_backup_schedule_task limit 20" + sql: "SELECT * FROM oceanbase.__all_virtual_backup_schedule_task where tenant_id=${tenant_id}" global: true - type: sql - sql: "SELECT * from oceanbase.CDB_OB_BACKUP_JOB_HISTORY where STATUS = 'FAILED' limit 20;" + sql: "SELECT * from oceanbase.CDB_OB_BACKUP_JOB_HISTORY where STATUS = 'FAILED' and tenant_id=${tenant_id};" global: true - type: sql - sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_DELETE_POLICY;" + sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_DELETE_POLICY where tenant_id=${tenant_id};" global: true - type: sql - sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_DELETE_JOBS limit 20" + sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_DELETE_JOBS where tenant_id=${tenant_id}" global: true - type: sql - sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_DELETE_TASKS limit 20" + sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_DELETE_TASKS where tenant_id=${tenant_id}" global: true - type: sql - sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_DELETE_TASK_HISTORY limit 20" + sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_DELETE_TASK_HISTORY where tenant_id=${tenant_id}" global: true - type: sql - sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_DELETE_JOB_HISTORY limit 20" + sql: "SELECT * FROM oceanbase.CDB_OB_BACKUP_DELETE_JOB_HISTORY where tenant_id=${tenant_id}" global: true - type: log global: false diff --git a/handler/gather/tasks/observer/io.yaml b/handler/gather/tasks/observer/io.yaml index fa4a488e..9e032539 100644 --- a/handler/gather/tasks/observer/io.yaml +++ b/handler/gather/tasks/observer/io.yaml @@ -82,7 +82,7 @@ task: sql: "show parameters like '%syslog_io_bandwidth_limit%';" global: true - type: sql - sql: "select * from oceanbase.__all_virtual_io_quota limit 20" + sql: "select * from oceanbase.__all_virtual_io_quota" global: true - type: ssh ssh: "df -h" diff --git a/handler/gather/tasks/observer/rootservice_switch.yaml b/handler/gather/tasks/observer/rootservice_switch.yaml index c0ea7f71..8afb6595 100644 --- a/handler/gather/tasks/observer/rootservice_switch.yaml +++ b/handler/gather/tasks/observer/rootservice_switch.yaml @@ -121,13 +121,13 @@ task: sql: "select * from oceanbase.DBA_OB_SERVER_EVENT_HISTORY where event like '%migrat%' and name6 like '%fail%' and value6=1;" global: true - type: sql - sql: "select * from oceanbase.DBA_OB_SERVER_EVENT_HISTORY where module='FAILURE_DETECTOR' limit 10;" + sql: "select * from oceanbase.DBA_OB_SERVER_EVENT_HISTORY where module='FAILURE_DETECTOR';" global: true - type: sql - sql: "select * from oceanbase.DBA_OB_SERVER_EVENT_HISTORY where module like '%ELECTION%' limit 10;" + sql: "select * from oceanbase.DBA_OB_SERVER_EVENT_HISTORY where module like '%ELECTION%';" global: true - type: sql - sql: "select * from oceanbase.GV$OB_LOG_STAT where role='LEADER' limit 20;" + sql: "select * from oceanbase.GV$OB_LOG_STAT where role='LEADER';" global: true - type: sql sql: "SELECT TENANT_NAME, TENANT_ID, TENANT_ROLE, STATUS, SWITCHOVER_STATUS FROM oceanbase.DBA_OB_TENANTS"