Added support for SQL diagnostic functionality.

.github/workflows/build_package.yml

@@ -83,10 +83,13 @@ jobs:
           sudo apt-get install -y alien
 
       - name: Convert RPM to DEB
-        run: sudo alien -k --scripts oceanbase-diagnostic-tool-*.rpm
+        run: |
+          sudo alien -k --scripts oceanbase-diagnostic-tool-*.rpm
+          pwd
 
       - name: Upload DEB Artifact
         uses: actions/upload-artifact@v3
         with:
           name: obdiag-deb-package
-          path: /home/runner/work/obdiag/obdiag/oceanbase-diagnostic-tool_*.deb
+          path: ./oceanbase-diagnostic-tool_*.deb
+          retention-days: 3

.github/workflows/test_sql_rule.yml

@@ -0,0 +1,30 @@
+name: Test Full Scan Rule
+
+on:
+  push:
+    branches: "*"
+  pull_request:
+    branches: "*"
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0 # Fetch all history for proper version detection
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.8
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements3.txt
+
+    - name: Run tests
+      run: python -m unittest discover -s test/analyzer/sql -p 'test_*.py'

.gitignore

@@ -1,4 +1,5 @@
 .idea/
+.vscode/
 venv/
 *.pyc
 *site-packages/

clean_all_result.sh

@@ -1,4 +1,4 @@
-rm -rf ./gather_pack_*
-rm -rf ./analyze_pack_*
-rm -rf ./analyze_flt_result*
-rm -rf ./check_report
+rm -rf ./obdiag_gather_pack_*
+rm -rf ./obdiag_analyze_pack_*
+rm -rf ./obdiag_analyze_flt_result*
+rm -rf ./obdiag_check_report

cmd.py

@@ -624,6 +624,49 @@ def _do_command(self, obdiag):
         return obdiag.analyze_fuction('analyze_flt_trace', self.opts)
 
 
+class ObdiagAnalyzeSQLCommand(ObdiagOriginCommand):
+
+    def __init__(self):
+        super(ObdiagAnalyzeSQLCommand, self).__init__('sql', 'Analyze oceanbase sql from sql_audit ')
+        self.parser.add_option('--host', type='string', help="tenant connection host")
+        self.parser.add_option('--port', type='string', help="tenant connection port")
+        self.parser.add_option('--password', type='string', help="tenant connection user password", default='')
+        self.parser.add_option('--user', type='string', help="tenant connection user name")
+        self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'")
+        self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'")
+        self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: <n> <m|h|d>. example: 1h.", default='30m')
+        self.parser.add_option('--level', type='string', help="The alarm level, optional parameters [critical, warn, notice, ok]", default='notice')
+        self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
+
+    def init(self, cmd, args):
+        super(ObdiagAnalyzeSQLCommand, self).init(cmd, args)
+        self.parser.set_usage('%s [options]' % self.prev_cmd)
+        return self
+
+    def _do_command(self, obdiag):
+        return obdiag.analyze_fuction('analyze_sql', self.opts)
+
+
+class ObdiagAnalyzeSQLReviewCommand(ObdiagOriginCommand):
+
+    def __init__(self):
+        super(ObdiagAnalyzeSQLReviewCommand, self).__init__('sql_review', 'Analyze oceanbase sql from sql_audit ')
+        self.parser.add_option('--host', type='string', help="tenant connection host")
+        self.parser.add_option('--port', type='string', help="tenant connection port")
+        self.parser.add_option('--password', type='string', help="tenant connection user password", default='')
+        self.parser.add_option('--user', type='string', help="tenant connection user name")
+        self.parser.add_option('--files', type='string', action="append", help="specify files")
+        self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
+
+    def init(self, cmd, args):
+        super(ObdiagAnalyzeSQLReviewCommand, self).init(cmd, args)
+        self.parser.set_usage('%s [options]' % self.prev_cmd)
+        return self
+
+    def _do_command(self, obdiag):
+        return obdiag.analyze_fuction('analyze_sql_review', self.opts)
+
+
 class ObdiagCheckCommand(ObdiagOriginCommand):
 
     def __init__(self):
@@ -748,6 +791,8 @@ def __init__(self):
         super(ObdiagAnalyzeCommand, self).__init__('analyze', 'Analyze oceanbase diagnostic info')
         self.register_command(ObdiagAnalyzeLogCommand())
         self.register_command(ObdiagAnalyzeFltTraceCommand())
+        self.register_command(ObdiagAnalyzeSQLCommand())
+        self.register_command(ObdiagAnalyzeSQLReviewCommand())
 
 
 class ObdiagRCACommand(MajorCommand):

common/ob_connector.py

@@ -46,6 +46,16 @@ def init(self):
         except Exception as e:
             self.stdio.verbose(e)
 
+    def __enter__(self):
+        """Ensures the database connection is open upon entering the 'with' block."""
+        self._connect_to_db()
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        """Automatically closes the database connection when exiting the 'with' block."""
+        if self.connection:
+            self.connection.close()
+
     def _connect_db(self):
         try:
             self.conn = mysql.connect(
@@ -82,17 +92,28 @@ def execute_sql(self, sql):
         cursor.close()
         return ret
 
-    def execute_sql_return_columns_and_data(self, sql):
+    def execute_sql_return_columns_and_data(self, sql, params=None):
+        """
+        Executes an SQL query and returns column names and data.
+
+        :param sql: The SQL statement to execute, using %s as a placeholder for parameters.
+        :param parameters: A tuple or list of parameters to substitute into the SQL statement.
+        :return: A tuple containing a list of column names and a list of rows (each a tuple).
+        """
         if self.conn is None:
             self._connect_db()
         else:
             self.conn.ping(reconnect=True)
-        cursor = self.conn.cursor()
-        cursor.execute(sql)
-        column_names = [col[0] for col in cursor.description]
-        ret = cursor.fetchall()
-        cursor.close()
-        return column_names, ret
+
+        with self.conn.cursor() as cursor:
+            if params:
+                cursor.execute(sql, params)
+            else:
+                cursor.execute(sql)
+
+            column_names = [col[0] for col in cursor.description]
+            data = cursor.fetchall()
+        return column_names, data
 
     def execute_sql_return_cursor_dictionary(self, sql):
         if self.conn is None:

common/tool.py

@@ -1396,3 +1396,59 @@ def get_nodes_list(context, nodes, stdio=None):
                 return None
             return new_nodes
         return None
+
+
+class SQLUtil(object):
+    re_trace = re.compile(r'''\/\*.*trace_id((?!\/\*).)*rpc_id.*\*\/''', re.VERBOSE)
+    re_annotation = re.compile(r'''\/\*((?!\/\*).)*\*\/''', re.VERBOSE)
+    re_interval = re.compile(
+        r'''interval\s?(\?|\-?\d+)\s?(day|hour|minute|second|microsecond|week|month|quarter|year|second_microsecond|minute_microsecond|minute_second|hour_microsecond|hour_second|hour_minute|day_microsecond|day_second|day_minute|day_hour|year_month)''',
+        re.VERBOSE,
+    )
+    re_force_index = re.compile(r'''force[\s]index[\s][(]\w+[)]''', re.VERBOSE)
+    re_cast_1 = re.compile(r'''cast\(.*?\(.*?\)\)''', re.VERBOSE)
+    re_cast_2 = re.compile(r'''cast\(.*?\)''', re.VERBOSE)
+    re_now = re.compile(r'''now\(\)''', re.VERBOSE)
+
+    def remove_sql_text_affects_parser(self, sql):
+        sql = sql.lower().strip()
+        sql = self.remove_hint_and_annotate(sql)
+        sql = self.remove_force_index(sql)
+        sql = self.remove_now_in_insert(sql)
+        sql = self.remove_semicolon(sql)
+        return sql
+
+    def remove_hint_and_annotate(self, sql):
+        sql = sql.lower()
+        sql = re.sub(self.re_annotation, '', sql)
+        sql = re.sub(self.re_trace, '', sql)
+        return sql
+
+    def replace_interval_day(self, sql):
+        sql = sql.lower()
+        sql = re.sub(self.re_interval, '?', sql)
+        return sql
+
+    def remove_force_index(self, sql):
+        sql = sql.lower()
+        sql = re.sub(self.re_force_index, '', sql)
+        return sql
+
+    def remove_cast(self, sql):
+        sql = sql.lower()
+        sql = re.sub(self.re_cast_1, '?', sql)
+        sql = re.sub(self.re_cast_2, '?', sql)
+        return sql
+
+    def remove_now_in_insert(self, sql):
+        sql = sql.lower().lstrip()
+        if sql.startswith('insert'):
+            sql = re.sub(self.re_now, '?', sql)
+        return sql
+
+    def remove_semicolon(self, sql):
+        sql = sql.strip()
+        return sql[:-1] if sql[-1] == ';' else sql
+
+    def get_db_id(self, database_alias, user_id):
+        return database_alias + '-' + user_id

conf/inner_config.yml

@@ -23,3 +23,11 @@ gather:
   scenes_base_path: "~/.obdiag/gather/tasks"
 rca:
   result_path: "./rca/"
+analyze_sql:
+  output_type: "html"
+  result_path: "./obdiag_analyze_sql/"
+  sql_audit_limit: 10
+  elapsed_time: 100
+analyze_sql_review:
+  output_type: "html"
+  result_path: "./obdiag_analyze_sql_review/"

config.py

@@ -91,6 +91,15 @@
     'rca': {
         'result_path': './rca/',
     },
+    'analyze_sql': {
+        'output_type': 'html',
+        'result_path': './obdiag_analyze_sql/',
+        'sql_audit_limit': 2000,
+    },
+    'analyze_sql_review': {
+        'output_type': 'html',
+        'result_path': './obdiag_analyze_sql_review/',
+    },
 }
 
 

core.py

@@ -31,6 +31,8 @@
 from err import CheckStatus, SUG_SSH_FAILED
 from handler.analyzer.analyze_flt_trace import AnalyzeFltTraceHandler
 from handler.analyzer.analyze_log import AnalyzeLogHandler
+from handler.analyzer.analyze_sql import AnalyzeSQLHandler
+from handler.analyzer.analyze_sql_review import AnalyzeSQLReviewHandler
 from handler.checker.check_handler import CheckHandler
 from handler.checker.check_list import CheckListHandler
 from handler.gather.gather_log import GatherLogHandler
@@ -267,6 +269,14 @@ def analyze_fuction(self, function_type, opt):
                 self.set_context(function_type, 'analyze', config)
                 handler = AnalyzeFltTraceHandler(self.context)
                 handler.handle()
+            elif function_type == 'analyze_sql':
+                self.set_context(function_type, 'analyze', config)
+                handler = AnalyzeSQLHandler(self.context)
+                handler.handle()
+            elif function_type == 'analyze_sql_review':
+                self.set_context(function_type, 'analyze', config)
+                handler = AnalyzeSQLReviewHandler(self.context)
+                handler.handle()
             else:
                 self._call_stdio('error', 'Not support analyze function: {0}'.format(function_type))
                 return False