obdiag实现参数、变量采集和比对功能

core.py

@@ -32,6 +32,8 @@
 from err import CheckStatus, SUG_SSH_FAILED
 from handler.analyzer.analyze_flt_trace import AnalyzeFltTraceHandler
 from handler.analyzer.analyze_log import AnalyzeLogHandler
+from handler.analyzer.analyze_parameter import AnalyzeParameterHandler
+from handler.analyzer.analyze_variable import AnalyzeVariableHandler
 from handler.analyzer.analyze_sql import AnalyzeSQLHandler
 from handler.analyzer.analyze_sql_review import AnalyzeSQLReviewHandler
 from handler.analyzer.analyze_parameter import AnalyzeParameterHandler
@@ -48,6 +50,8 @@
 from handler.gather.gather_plan_monitor import GatherPlanMonitorHandler
 from handler.gather.gather_scenes import GatherSceneHandler
 from handler.gather.scenes.list import GatherScenesListHandler
+from handler.gather.gather_parameters import GatherParametersHandler
+from handler.gather.gather_variables import GatherVariablesHandler
 from handler.gather.gather_tabledump import GatherTableDumpHandler
 from handler.gather.gather_parameters import GatherParametersHandler
 from handler.gather.gather_variables import GatherVariablesHandler
@@ -246,6 +250,12 @@ def gather_function(self, function_type, opt):
             elif function_type == 'gather_variables':
                 handler = GatherVariablesHandler(self.context)
                 return handler.handle()
+            elif function_type == 'gather_parameters':
+                handler = GatherParametersHandler(self.context)
+                return handler.handle()
+            elif function_type == 'gather_variables':
+                handler = GatherVariablesHandler(self.context)
+                return handler.handle()
             else:
                 self._call_stdio('error', 'Not support gather function: {0}'.format(function_type))
                 return False
@@ -284,6 +294,18 @@ def analyze_fuction(self, function_type, opt):
                 self.set_context(function_type, 'analyze', config)
                 handler = AnalyzeFltTraceHandler(self.context)
                 handler.handle()
+            elif function_type == 'analyze_parameter_default':
+                self.set_context(function_type, 'analyze', config)
+                handler = AnalyzeParameterHandler(self.context, 'default')
+                handler.handle()
+            elif function_type == 'analyze_parameter_diff':
+                self.set_context_skip_cluster_conn(function_type, 'analyze', config)
+                handler = AnalyzeParameterHandler(self.context, 'diff')
+                handler.handle()
+            elif function_type == 'analyze_variable_diff':
+                self.set_context(function_type, 'analyze', config)
+                handler = AnalyzeVariableHandler(self.context, 'diff')
+                handler.handle()
             elif function_type == 'analyze_sql':
                 self.set_context(function_type, 'analyze', config)
                 handler = AnalyzeSQLHandler(self.context)

diag_cmd.py

@@ -697,20 +697,49 @@ def _do_command(self, obdiag):
         return obdiag.analyze_fuction('analyze_parameter_diff', self.opts)
 
 
-class ObdiagAnalyzeParameterNonDefaultCommand(ObdiagOriginCommand):
+class ObdiagAnalyzeParameterDefaultCommand(ObdiagOriginCommand):
     def __init__(self):
-        super(ObdiagAnalyzeParameterNonDefaultCommand, self).__init__('non-default', 'Analyze the parameter to identify parameters with non-default values')
+        super(ObdiagAnalyzeParameterDefaultCommand, self).__init__('default', 'Analyze the parameter to identify parameters with non-default values')
         self.parser.add_option('--file', type='string', help="specify initialization parameter file")
         self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
 
     def init(self, cmd, args):
-        super(ObdiagAnalyzeParameterNonDefaultCommand, self).init(cmd, args)
+        super(ObdiagAnalyzeParameterDefaultCommand, self).init(cmd, args)
         self.parser.set_usage('%s [options]' % self.prev_cmd)
         return self
 
     def _do_command(self, obdiag):
-        return obdiag.analyze_fuction('analyze_parameter_non_default', self.opts)
+        return obdiag.analyze_fuction('analyze_parameter_default', self.opts)
+
+
+class ObdiagAnalyzeParameterCommand(MajorCommand):
+    def __init__(self):
+        super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info')
+        self.register_command(ObdiagAnalyzeParameterDiffCommand())
+        self.register_command(ObdiagAnalyzeParameterDefaultCommand())
+
+
+class ObdiagAnalyzeVariableDiffCommand(ObdiagOriginCommand):
+    def __init__(self):
+        super(ObdiagAnalyzeVariableDiffCommand, self).__init__('diff', 'Analyze and identify variables that have changed compared to the specified variable file')
+        self.parser.add_option('--file', type='string', help="specify initialization parameter file")
+        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
+        self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
+
+    def init(self, cmd, args):
+        super(ObdiagAnalyzeVariableDiffCommand, self).init(cmd, args)
+        self.parser.set_usage('%s [options]' % self.prev_cmd)
+        return self
+
+    def _do_command(self, obdiag):
+        return obdiag.analyze_fuction('analyze_variable_diff', self.opts)
+
+
+class ObdiagAnalyzeVariableCommand(MajorCommand):
+    def __init__(self):
+        super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze oceanbase variables info')
+        self.register_command(ObdiagAnalyzeVariableDiffCommand())
 
 
 class ObdiagAnalyzeSQLCommand(ObdiagOriginCommand):
@@ -764,29 +793,6 @@ def _do_command(self, obdiag):
         return obdiag.analyze_fuction('analyze_sql_review', self.opts)
 
 
-class ObdiagAnalyzeParameterCommand(MajorCommand):
-    def __init__(self):
-        super(ObdiagAnalyzeParameterCommand, self).__init__('parameter', 'Analyze oceanbase parameters info')
-        self.register_command(ObdiagAnalyzeParameterDiffCommand())
-        self.register_command(ObdiagAnalyzeParameterNonDefaultCommand())
-
-
-class ObdiagAnalyzeVariableCommand(ObdiagOriginCommand):
-    def __init__(self):
-        super(ObdiagAnalyzeVariableCommand, self).__init__('variable', 'Analyze and identify variables that have changed compared to the specified variable file')
-        self.parser.add_option('--file', type='string', help="specify initialization parameter file")
-        self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
-        self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
-
-    def init(self, cmd, args):
-        super(ObdiagAnalyzeVariableCommand, self).init(cmd, args)
-        self.parser.set_usage('%s [options]' % self.prev_cmd)
-        return self
-
-    def _do_command(self, obdiag):
-        return obdiag.analyze_fuction('analyze_variable', self.opts)
-
-
 class ObdiagCheckCommand(ObdiagOriginCommand):
 
     def __init__(self):
@@ -895,6 +901,8 @@ def __init__(self):
         self.register_command(ObdiagGatherObproxyLogCommand())
         self.register_command(ObdiagGatherSceneCommand())
         self.register_command(ObdiagGatherAshReportCommand())
+        self.register_command(ObdiagGatherParameterCommand())
+        self.register_command(ObdiagGatherVariableCommand())
         self.register_command(ObdiagGatherTableDumpHandler())
         self.register_command(ObdiagGatherParameterCommand())
         self.register_command(ObdiagGatherVariableCommand())
@@ -914,6 +922,8 @@ def __init__(self):
         super(ObdiagAnalyzeCommand, self).__init__('analyze', 'Analyze oceanbase diagnostic info')
         self.register_command(ObdiagAnalyzeLogCommand())
         self.register_command(ObdiagAnalyzeFltTraceCommand())
+        self.register_command(ObdiagAnalyzeParameterCommand())
+        self.register_command(ObdiagAnalyzeVariableCommand())
         self.register_command(ObdiagAnalyzeSQLCommand())
         self.register_command(ObdiagAnalyzeSQLReviewCommand())
         self.register_command(ObdiagAnalyzeParameterCommand())

handler/analyzer/analyze_parameter.py

@@ -28,7 +28,7 @@
 
 
 class AnalyzeParameterHandler(object):
-    def __init__(self, context, analyze_type='non_default'):
+    def __init__(self, context, analyze_type='default'):
         self.context = context
         self.stdio = self.context.stdio
         self.export_report_path = None
@@ -59,13 +59,13 @@ def get_version(self):
         try:
             observer_version = get_observer_version_by_sql(self.ob_cluster, self.stdio)
         except Exception as e:
-            self.stdio.warn("failed to get observer version:{0}".format(e))
-        self.stdio.verbose("get observer version: {0}".format(observer_version))
+            self.stdio.warn("AnalyzeHandler Failed to get observer version:{0}".format(e))
+        self.stdio.verbose("AnalyzeHandler.init get observer version: {0}".format(observer_version))
         return observer_version
 
     def handle(self):
-        if self.analyze_type == 'non_default':
-            if not self.init_option_non_default():
+        if self.analyze_type == 'default':
+            if not self.init_option_default():
                 self.stdio.error('init option failed')
                 return False
         else:
@@ -76,13 +76,13 @@ def handle(self):
         DirectoryUtil.mkdir(path=self.export_report_path, stdio=self.stdio)
         self.execute()
 
-    def init_option_non_default(self):
+    def init_option_default(self):
         options = self.context.options
         store_dir_option = Util.get_option(options, 'store_dir')
         offline_file_option = Util.get_option(options, 'file')
         if store_dir_option and store_dir_option != "./":
             if not os.path.exists(os.path.abspath(store_dir_option)):
-                self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
+                self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
                 os.makedirs(os.path.abspath(store_dir_option))
             self.export_report_path = os.path.abspath(store_dir_option)
         else:
@@ -105,7 +105,7 @@ def init_option_diff(self):
         offline_file_option = Util.get_option(options, 'file')
         if store_dir_option and store_dir_option != "./":
             if not os.path.exists(os.path.abspath(store_dir_option)):
-                self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
+                self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
                 os.makedirs(os.path.abspath(store_dir_option))
             self.export_report_path = os.path.abspath(store_dir_option)
         else:
@@ -123,7 +123,7 @@ def init_option_diff(self):
                 self.parameter_file_name = os.path.abspath(offline_file_option)
         return True
 
-    def analyze_parameter_non_default(self):
+    def analyze_parameter_default(self):
         observer_version = self.get_version()
         if StringUtils.compare_versions_greater(observer_version, "4.2.2.0"):
             if self.parameter_file_name is not None:
@@ -144,7 +144,7 @@ def analyze_parameter_non_default(self):
                 report_default_tb.add_row([row[1], row[2], row[3], row[4], tenant_id, row[6], row[11], row[7]])
             fp.write(report_default_tb.get_string() + "\n")
             self.stdio.print(report_default_tb.get_string())
-            self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'")
+            self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0}' ".format(file_name) + Style.RESET_ALL + "'")
         else:
             if self.parameter_file_name is None:
                 self.stdio.error("the version of OceanBase is lower than 4.2.2, an initialization parameter file must be provided to find non-default values")
@@ -179,9 +179,9 @@ def analyze_parameter_non_default(self):
                 fp.write(report_default_tb.get_string() + "\n")
                 if not is_empty:
                     self.stdio.print(report_default_tb.get_string())
-                    self.stdio.print("Analyze parameter non-default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'")
+                    self.stdio.print("Analyze parameter default finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(file_name) + Style.RESET_ALL + "'")
                 else:
-                    self.stdio.print("Analyze parameter non-default finished. All parameter values are the same as the default values.")
+                    self.stdio.print("Analyze parameter default finished. All parameter values are the same as the default values.")
 
     def alalyze_parameter_diff(self):
         if self.parameter_file_name is None:
@@ -258,8 +258,8 @@ def alalyze_parameter_diff(self):
 
     def execute(self):
         try:
-            if self.analyze_type == 'non_default':
-                self.analyze_parameter_non_default()
+            if self.analyze_type == 'default':
+                self.analyze_parameter_default()
             elif self.analyze_type == 'diff':
                 self.alalyze_parameter_diff()
         except Exception as e:

handler/analyzer/analyze_variable.py

@@ -26,11 +26,12 @@
 
 
 class AnalyzeVariableHandler(object):
-    def __init__(self, context):
+    def __init__(self, context, analyze_type='diff'):
         self.context = context
         self.stdio = self.context.stdio
         self.export_report_path = None
         self.variable_file_name = None
+        self.analyze_type = analyze_type
         self.ob_cluster = self.context.cluster_config
         if self.context.get_variable("gather_timestamp", None):
             self.analyze_timestamp = self.context.get_variable("gather_timestamp")
@@ -48,8 +49,8 @@ def __init__(self, context):
                 database="oceanbase",
             )
         except Exception as e:
-            self.stdio.error("failed to connect to database: {0}".format(e))
-            raise OBDIAGFormatException("failed to connect to database: {0}".format(e))
+            self.stdio.error("Failed to connect to database: {0}".format(e))
+            raise OBDIAGFormatException("Failed to connect to database: {0}".format(e))
 
     def handle(self):
         if not self.init_option():
@@ -75,7 +76,7 @@ def init_option(self):
 
         if store_dir_option and store_dir_option != "./":
             if not os.path.exists(os.path.abspath(store_dir_option)):
-                self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
+                self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
                 os.makedirs(os.path.abspath(store_dir_option))
             self.export_report_path = os.path.abspath(store_dir_option)
         else:
@@ -87,7 +88,7 @@ def init_option(self):
 
         return True
 
-    def alalyze_variable(self):
+    def analyze_variable(self):
         sql = '''select version(), tenant_id, zone, name,gmt_modified, value, flags, min_val, max_val, now() 
         from oceanbase.__all_virtual_sys_variable order by 2, 4, 5'''
         db_variable_info = self.obconn.execute_sql(sql)
@@ -131,6 +132,6 @@ def alalyze_variable(self):
 
     def execute(self):
         try:
-            self.alalyze_variable()
+            self.analyze_variable()
         except Exception as e:
             self.stdio.error("variable info analyze failed, error message: {0}".format(e))

handler/gather/gather_parameters.py

@@ -47,8 +47,8 @@ def __init__(self, context, gather_pack_dir='./'):
                 database="oceanbase",
             )
         except Exception as e:
-            self.stdio.error("failed to connect to database: {0}".format(e))
-            raise OBDIAGFormatException("failed to connect to database: {0}".format(e))
+            self.stdio.error("Failed to connect to database: {0}".format(e))
+            raise OBDIAGFormatException("Failed to connect to database: {0}".format(e))
 
     def handle(self):
         if not self.init_option():
@@ -66,7 +66,7 @@ def init_option(self):
         store_dir_option = Util.get_option(options, 'store_dir')
         if store_dir_option and store_dir_option != "./":
             if not os.path.exists(os.path.abspath(store_dir_option)):
-                self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
+                self.stdio.warn('warn: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
                 os.makedirs(os.path.abspath(store_dir_option))
         self.gather_pack_dir = os.path.abspath(store_dir_option)
         return True
@@ -76,8 +76,8 @@ def get_version(self):
         try:
             observer_version = get_observer_version_by_sql(self.ob_cluster, self.stdio)
         except Exception as e:
-            self.stdio.warn("failed to get observer version:{0}".format(e))
-        self.stdio.verbose("get observer version: {0}".format(observer_version))
+            self.stdio.warn("GatherHandler Failed to get observer version:{0}".format(e))
+        self.stdio.verbose("GatherHandler.init get observer version: {0}".format(observer_version))
         return observer_version
 
     def get_cluster_name(self):
@@ -87,8 +87,8 @@ def get_cluster_name(self):
             cluster_info = self.obconn.execute_sql(sql)
             cluster_name = cluster_info[0][0]
         except Exception as e:
-            self.stdio.warn("failed to get oceanbase cluster name:{0}".format(e))
-        self.stdio.verbose("get oceanbase cluster name {0}".format(cluster_name))
+            self.stdio.warn("RCAHandler Failed to get oceanbase cluster name:{0}".format(e))
+        self.stdio.verbose("RCAHandler.init get oceanbase cluster name {0}".format(cluster_name))
         return cluster_name
 
     def get_parameters_info(self):
@@ -110,8 +110,10 @@ def get_parameters_info(self):
 '''
             parameter_info = self.obconn.execute_sql(sql)
             self.parameter_file_name = self.gather_pack_dir + '/{0}_parameters_{1}.csv'.format(cluster_name, TimeUtils.timestamp_to_filename_time(self.gather_timestamp))
+            header = ['VERSION', 'SVR_IP', 'SVR_PORT', 'ZONE', 'SCOPE', 'TENANT_ID', 'NAME', 'VALUE', 'SECTION', 'EDIT_LEVEL', 'RECORD_TIME', 'DEFAULT_VALUE', 'ISDEFAULT']
             with open(self.parameter_file_name, 'w', newline='') as file:
                 writer = csv.writer(file)
+                writer.writerow(header)
                 for row in parameter_info:
                     if row[5] is None:
                         tmp_row = [col for col in row]
@@ -121,7 +123,7 @@ def get_parameters_info(self):
                         writer.writerow(row)
             self.stdio.print("Gather parameters finished. For more details, please run cmd '" + Fore.YELLOW + "cat {0}".format(self.parameter_file_name) + Style.RESET_ALL + "'")
         else:
-            self.stdio.warn("failed to retrieve the database version. Please check if the database connection is normal.")
+            self.stdio.warn("Failed to retrieve the database version. Please check if the database connection is normal.")
 
     def execute(self):
         try: