From c57f281d005b8490fe4de1935302a66f27c5fdb3 Mon Sep 17 00:00:00 2001 From: oraclebird <534223814@qq.com> Date: Mon, 15 Jul 2024 18:28:13 +0800 Subject: [PATCH] obdiag gather and analyze parameters/variables --- core.py | 8 +-- diag_cmd.py | 73 +++++---------------------- handler/analyzer/analyze_parameter.py | 20 ++++---- handler/analyzer/analyze_variable.py | 7 +-- handler/gather/gather_parameters.py | 17 ++----- handler/gather/gather_variables.py | 3 +- 6 files changed, 36 insertions(+), 92 deletions(-) diff --git a/core.py b/core.py index ec26f1a7..738cd1c1 100644 --- a/core.py +++ b/core.py @@ -294,17 +294,17 @@ def analyze_fuction(self, function_type, opt): self.set_context(function_type, 'analyze', config) handler = AnalyzeFltTraceHandler(self.context) handler.handle() - elif function_type == 'analyze_parameter_non_default': + elif function_type == 'analyze_parameter_default': self.set_context(function_type, 'analyze', config) - handler = AnalyzeParameterHandler(self.context, 'non_default') + handler = AnalyzeParameterHandler(self.context, 'default') handler.handle() elif function_type == 'analyze_parameter_diff': self.set_context_skip_cluster_conn(function_type, 'analyze', config) handler = AnalyzeParameterHandler(self.context, 'diff') handler.handle() - elif function_type == 'analyze_variable': + elif function_type == 'analyze_variable_diff': self.set_context(function_type, 'analyze', config) - handler = AnalyzeVariableHandler(self.context) + handler = AnalyzeVariableHandler(self.context, 'diff') handler.handle() elif function_type == 'analyze_sql': self.set_context(function_type, 'analyze', config) diff --git a/diag_cmd.py b/diag_cmd.py index f2970dac..cc54650a 100644 --- a/diag_cmd.py +++ b/diag_cmd.py @@ -697,75 +697,49 @@ def _do_command(self, obdiag): return obdiag.analyze_fuction('analyze_parameter_diff', self.opts) -class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand): +class ObdiagAnalyzeParameterDefaultCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'Analyze the parameter to identify parameters with non-default values') + super(ObdiagAnalyzeParameterDefaultCommand, self).__init__('default', 'Analyze the parameter to identify parameters with non-default values') self.parser.add_option('--file', type='string', help="specify initialization parameter file") self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) def init(self, cmd, args): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args) + super(ObdiagAnalyzeParameterDefaultCommand, self).init(cmd, args) self.parser.set_usage('%s [options]' % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts) + return obdiag.analyze_fuction('analyze_parameter_default', self.opts) class ObdiagAnalyzeParameterCommand(MajorCommand): def __init__(self): super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info') self.register_command(ObdiagAnalyzeParameterDiffCommand()) - self.register_command(ObdiagAnalyzeParameterNonDefaultCommand()) + self.register_command(ObdiagAnalyzeParameterDefaultCommand()) -class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand): +class ObdiagAnalyzeVariableDiffCommand(ObdiagOriginCommand): def __init__(self): - super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze and identify variables that have changed compared to the specified variable file') + super(ObdiagAnalyzeVariableDiffCommand, self).__init__('diff', 'Analyze and identify variables that have changed compared to the specified variable file') self.parser.add_option('--file', type='string', help="specify initialization parameter file") self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) def init(self, cmd, args): - super(ObdiagAnalyzeVariableCommand, self).init(cmd, args) + super(ObdiagAnalyzeVariableDiffCommand, self).init(cmd, args) self.parser.set_usage('%s [options]' % self.prev_cmd) return self def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_variable', self.opts) + return obdiag.analyze_fuction('analyze_variable_diff', self.opts) -class ObdiagAnalyzeParameterDiffCommand(ObdiagOriginCommand): - def __init__(self): - super(ObdiagAnalyzeParameterDiffCommand, self).__init__('diff', 'Analyze the parameter configurations between observers and identify the parameters with different values among the observers') - self.parser.add_option('--file', type='string', help="specify initialization parameter file") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeParameterDiffCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_parameter_diff', self.opts) - - -class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand): +class ObdiagAnalyzeVariableCommand(MajorCommand): def __init__(self): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'Analyze the parameter to identify parameters with non-default values') - self.parser.add_option('--file', type='string', help="specify initialization parameter file") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts) + super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze oceanbase variables info') + self.register_command(ObdiagAnalyzeVariableDiffCommand()) class ObdiagAnalyzeSQLCommand(ObdiagOriginCommand): @@ -819,29 +793,6 @@ def _do_command(self, obdiag): return obdiag.analyze_fuction('analyze_sql_review', self.opts) -class ObdiagAnalyzeParameterCommand(MajorCommand): - def __init__(self): - super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info') - self.register_command(ObdiagAnalyzeParameterDiffCommand()) - self.register_command(ObdiagAnalyzeParameterNonDefaultCommand()) - - -class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand): - def __init__(self): - super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze and identify variables that have changed compared to the specified variable file') - self.parser.add_option('--file', type='string', help="specify initialization parameter file") - self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') - self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) - - def init(self, cmd, args): - super(ObdiagAnalyzeVariableCommand, self).init(cmd, args) - self.parser.set_usage('%s [options]' % self.prev_cmd) - return self - - def _do_command(self, obdiag): - return obdiag.analyze_fuction('analyze_variable', self.opts) - - class ObdiagCheckCommand(ObdiagOriginCommand): def __init__(self): diff --git a/handler/analyzer/analyze_parameter.py b/handler/analyzer/analyze_parameter.py index 97ca13cc..475998d2 100644 --- a/handler/analyzer/analyze_parameter.py +++ b/handler/analyzer/analyze_parameter.py @@ -28,7 +28,7 @@ class AnalyzeParameterHandler(object): - def __init__(self, context, analyze_type='non_default'): + def __init__(self, context, analyze_type='default'): self.context = context self.stdio = self.context.stdio self.export_report_path = None @@ -64,8 +64,8 @@ def get_version(self): return observer_version def handle(self): - if self.analyze_type == 'non_default': - if not self.init_option_non_default(): + if self.analyze_type == 'default': + if not self.init_option_default(): self.stdio.error('init option failed') return False else: @@ -76,7 +76,7 @@ def handle(self): DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio) self.execute() - def init_option_non_default(self): + def init_option_default(self): options = self.context.options store_dir_option = Util.get_option(options, 'store_dir') offline_file_option = Util.get_option(options, 'file') @@ -123,7 +123,7 @@ def init_option_diff(self): self.parameter_file_name = os.path.abspath(offline_file_option) return True - def analyze_parameter_non_default(self): + def analyze_parameter_default(self): observer_version = self.get_version() if StringUtils.compare_versions_greater(observer_version, "4.2.2.0"): if self.parameter_file_name is not None: @@ -144,7 +144,7 @@ def analyze_parameter_non_default(self): report_default_tb.add_row([row[1], row[2], row[3], row[4], tenant_id, row[6], row[11], row[7]]) fp.write(report_default_tb.get_string() + "\n") self.stdio.print(report_default_tb.get_string()) - self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'") + self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'") else: if self.parameter_file_name is None: self.stdio.error("the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values") @@ -179,9 +179,9 @@ def analyze_parameter_non_default(self): fp.write(report_default_tb.get_string() + "\n") if not is_empty: self.stdio.print(report_default_tb.get_string()) - self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") + self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'") else: - self.stdio.print("Analyze parameter non-default finished. All parameter values are the same as the default values.") + self.stdio.print("Analyze parameter default finished. All parameter values are the same as the default values.") def alalyze_parameter_diff(self): if self.parameter_file_name is None: @@ -258,8 +258,8 @@ def alalyze_parameter_diff(self): def execute(self): try: - if self.analyze_type == 'non_default': - self.analyze_parameter_non_default() + if self.analyze_type == 'default': + self.analyze_parameter_default() elif self.analyze_type == 'diff': self.alalyze_parameter_diff() except Exception as e: diff --git a/handler/analyzer/analyze_variable.py b/handler/analyzer/analyze_variable.py index 43fc8d32..ac86050d 100644 --- a/handler/analyzer/analyze_variable.py +++ b/handler/analyzer/analyze_variable.py @@ -26,11 +26,12 @@ class AnalyzeVariableHandler(object): - def __init__(self, context): + def __init__(self, context, analyze_type='diff'): self.context = context self.stdio = self.context.stdio self.export_report_path = None self.variable_file_name = None + self.analyze_type = analyze_type self.ob_cluster = self.context.cluster_config if self.context.get_variable("gather_timestamp", None): self.analyze_timestamp = self.context.get_variable("gather_timestamp") @@ -87,7 +88,7 @@ def init_option(self): return True - def alalyze_variable(self): + def analyze_variable(self): sql = '''select version(), tenant_id, zone, name,gmt_modified, value, flags, min_val, max_val, now() from oceanbase.__all_virtual_sys_variable order by 2, 4, 5''' db_variable_info = self.obconn.execute_sql(sql) @@ -131,6 +132,6 @@ def alalyze_variable(self): def execute(self): try: - self.alalyze_variable() + self.analyze_variable() except Exception as e: self.stdio.error("variable info analyze failed, error message: {0}".format(e)) diff --git a/handler/gather/gather_parameters.py b/handler/gather/gather_parameters.py index ff9ac5a8..75bc4286 100644 --- a/handler/gather/gather_parameters.py +++ b/handler/gather/gather_parameters.py @@ -66,8 +66,7 @@ def init_option(self): store_dir_option = Util.get_option(options, 'store_dir') if store_dir_option and store_dir_option != "./": if not os.path.exists(os.path.abspath(store_dir_option)): - self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format( - os.path.abspath(store_dir_option))) + self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) os.makedirs(os.path.abspath(store_dir_option)) self.gather_pack_dir = os.path.abspath(store_dir_option) return True @@ -110,11 +109,8 @@ def get_parameters_info(self): EDIT_LEVEL, now(), '','' from oceanbase.__all_virtual_sys_parameter_stat where scope='CLUSTER' ''' parameter_info = self.obconn.execute_sql(sql) - self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, - TimeUtils.timestamp_to_filename_time( - self.gather_timestamp)) - header = ['VERSION', 'SVR_IP', 'SVR_PORT', 'ZONE', 'SCOPE', 'TENANT_ID', 'NAME', 'VALUE', 'SECTION', - 'EDIT_LEVEL', 'RECORD_TIME', 'DEFAULT_VALUE', 'ISDEFAULT'] + self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) + header = ['VERSION', 'SVR_IP', 'SVR_PORT', 'ZONE', 'SCOPE', 'TENANT_ID', 'NAME', 'VALUE', 'SECTION', 'EDIT_LEVEL', 'RECORD_TIME', 'DEFAULT_VALUE', 'ISDEFAULT'] with open(self.parameter_file_name, 'w', newline='') as file: writer = csv.writer(file) writer.writerow(header) @@ -125,12 +121,9 @@ def get_parameters_info(self): writer.writerow(tmp_row) else: writer.writerow(row) - self.stdio.print( - "Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format( - self.parameter_file_name) + Style.RESET_ALL + "'") + self.stdio.print("Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.parameter_file_name) + Style.RESET_ALL + "'") else: - self.stdio.warn( - "Failed to retrieve the database version. Please check if the database connection is normal.") + self.stdio.warn("Failed to retrieve the database version. Please check if the database connection is normal.") def execute(self): try: diff --git a/handler/gather/gather_variables.py b/handler/gather/gather_variables.py index 82f090ff..56f7ef20 100644 --- a/handler/gather/gather_variables.py +++ b/handler/gather/gather_variables.py @@ -86,8 +86,7 @@ def get_variables_info(self): from oceanbase.__all_virtual_sys_variable order by 2, 4, 5''' variable_info = self.obconn.execute_sql(sql) self.variable_file_name = self.gather_pack_dir + '/{0}_variables_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp)) - header = ['VERSION', 'TENANT_ID', 'ZONE', 'NAME', 'GMT_MODIFIED', 'VALUE', 'FLAGS', - 'MIN_VALUE', 'MAX_VALUE', 'RECORD_TIME'] + header = ['VERSION', 'TENANT_ID', 'ZONE', 'NAME', 'GMT_MODIFIED', 'VALUE', 'FLAGS', 'MIN_VALUE', 'MAX_VALUE', 'RECORD_TIME'] with open(self.variable_file_name, 'w', newline='') as file: writer = csv.writer(file) writer.writerow(header)