From 8cc1cb8173de57b224208b8b8f475dafd12470e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 16 Dec 2024 20:37:52 +0800 Subject: [PATCH 1/6] fix: conf path --- rpm/oceanbase-diagnostic-tool.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpm/oceanbase-diagnostic-tool.spec b/rpm/oceanbase-diagnostic-tool.spec index c9828b6..82d6103 100644 --- a/rpm/oceanbase-diagnostic-tool.spec +++ b/rpm/oceanbase-diagnostic-tool.spec @@ -46,7 +46,7 @@ cd $SRC_DIR \cp -rf $SRC_DIR/rpm/init.sh $BUILD_DIR/SOURCES/init.sh \cp -rf $SRC_DIR/rpm/init_obdiag_cmd.sh $BUILD_DIR/SOURCES/init_obdiag_cmd.sh \cp -rf $SRC_DIR/rpm/obdiag_backup.sh $BUILD_DIR/SOURCES/obdiag_backup.sh -\cp -rf $SRC_DIR/conf $BUILD_DIR/SOURCES/conf +\cp -rf $SRC_DIR/conf/* $BUILD_DIR/SOURCES/conf/ mkdir -p ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/lib/ mkdir -p ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/dependencies/bin find $SRC_DIR -name "obdiag" @@ -55,7 +55,7 @@ find $SRC_DIR -name "obdiag" \cp -rf $BUILD_DIR/SOURCES/resources ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/resources \cp -rf $BUILD_DIR/SOURCES/dependencies/bin ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/dependencies \cp -rf $BUILD_DIR/SOURCES/example ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/ -\cp -rf $BUILD_DIR/SOURCES/conf ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/ +\cp -rf $BUILD_DIR/SOURCES/conf ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/conf \cp -rf $BUILD_DIR/SOURCES/init.sh ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/ \cp -rf $BUILD_DIR/SOURCES/init_obdiag_cmd.sh ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/ \cp -rf $BUILD_DIR/SOURCES/obdiag_backup.sh ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/ From 51e759b7aa50e2f35dbc84d686470ccf44c8cd7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 16 Dec 2024 21:07:16 +0800 Subject: [PATCH 2/6] fix: conf path --- rpm/oceanbase-diagnostic-tool.spec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpm/oceanbase-diagnostic-tool.spec b/rpm/oceanbase-diagnostic-tool.spec index 82d6103..a55f87c 100644 --- a/rpm/oceanbase-diagnostic-tool.spec +++ b/rpm/oceanbase-diagnostic-tool.spec @@ -42,11 +42,11 @@ cd $SRC_DIR \cp -rf $SRC_DIR/example $BUILD_DIR/SOURCES/example \cp -rf $SRC_DIR/resources $BUILD_DIR/SOURCES/ \cp -rf $SRC_DIR/dependencies/bin $BUILD_DIR/SOURCES/dependencies -\cp -rf $SRC_DIR/plugins $BUILD_DIR/plugins +\cp -rf $SRC_DIR/plugins $BUILD_DIR/SOURCES/ \cp -rf $SRC_DIR/rpm/init.sh $BUILD_DIR/SOURCES/init.sh \cp -rf $SRC_DIR/rpm/init_obdiag_cmd.sh $BUILD_DIR/SOURCES/init_obdiag_cmd.sh \cp -rf $SRC_DIR/rpm/obdiag_backup.sh $BUILD_DIR/SOURCES/obdiag_backup.sh -\cp -rf $SRC_DIR/conf/* $BUILD_DIR/SOURCES/conf/ +\cp -rf $SRC_DIR/conf $BUILD_DIR/SOURCES/ mkdir -p ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/lib/ mkdir -p ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/dependencies/bin find $SRC_DIR -name "obdiag" @@ -59,7 +59,7 @@ find $SRC_DIR -name "obdiag" \cp -rf $BUILD_DIR/SOURCES/init.sh ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/ \cp -rf $BUILD_DIR/SOURCES/init_obdiag_cmd.sh ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/ \cp -rf $BUILD_DIR/SOURCES/obdiag_backup.sh ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/ -\cp -rf $BUILD_DIR/plugins ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/ +\cp -rf $BUILD_DIR/SOURCES/plugins ${RPM_BUILD_ROOT}/usr/local/oceanbase-diagnostic-tool/ %files From ba72c0cb6a5a1af1db6216620fa9abd2ad66a7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Wed, 18 Dec 2024 15:33:16 +0800 Subject: [PATCH 3/6] remote_client support strict_host_key_checking --- conf/inner_config.yml | 1 + rpm/init_obdiag_cmd.sh | 3 +++ src/common/config.py | 1 + src/common/ssh_client/remote_client.py | 13 ++++++++++--- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/conf/inner_config.yml b/conf/inner_config.yml index 7ee5484..cdca7e6 100644 --- a/conf/inner_config.yml +++ b/conf/inner_config.yml @@ -5,6 +5,7 @@ obdiag: file_number_limit: 50 file_size_limit: 5G dis_rsa_algorithms: 0 + strict_host_key_checking: 0 logger: log_dir: ~/.obdiag/log log_filename: obdiag.log diff --git a/rpm/init_obdiag_cmd.sh b/rpm/init_obdiag_cmd.sh index c9d3ca5..99a3ce6 100644 --- a/rpm/init_obdiag_cmd.sh +++ b/rpm/init_obdiag_cmd.sh @@ -10,6 +10,9 @@ _obdiag_completion() { ;; 2) case "${COMP_WORDS[1]}" in + check) + type_list="run list" + ;; gather) if [ "$COMP_CWORD" -eq 2 ]; then type_list="log clog slog plan_monitor stack perf sysstat obproxy_log all scene ash tabledump parameter variable" diff --git a/src/common/config.py b/src/common/config.py index 77690af..45e12cb 100644 --- a/src/common/config.py +++ b/src/common/config.py @@ -70,6 +70,7 @@ 'file_number_limit': 20, 'file_size_limit': '2G', 'dis_rsa_algorithms': 0, + 'strict_host_key_checking': 0, }, 'logger': { 'log_dir': '~/.obdiag/log', diff --git a/src/common/ssh_client/remote_client.py b/src/common/ssh_client/remote_client.py index e046c62..a96d69b 100644 --- a/src/common/ssh_client/remote_client.py +++ b/src/common/ssh_client/remote_client.py @@ -58,23 +58,30 @@ def __init__(self, context, node): remote_client_disable_rsa_algorithms = bool(self.context.inner_config.get("obdiag").get("basic").get("dis_rsa_algorithms")) if remote_client_disable_rsa_algorithms: self._disabled_rsa_algorithms = DISABLED_ALGORITHMS + remote_client_missing_host_key_policy = bool(self.context.inner_config.get("obdiag").get("basic").get("strict_host_key_checking")) self.ssh_type = "remote" if len(self.key_file) > 0: try: self._ssh_fd = paramiko.SSHClient() + if remote_client_missing_host_key_policy: + self._ssh_fd.set_missing_host_key_policy(paramiko.MissingHostKeyPolicy()) + else: + self._ssh_fd.load_system_host_keys() self._ssh_fd.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) - self._ssh_fd.load_system_host_keys() self._ssh_fd.connect(hostname=self.host_ip, username=self.username, key_filename=self.key_file, port=self.ssh_port, disabled_algorithms=self._disabled_rsa_algorithms) except AuthenticationException: self.password = input("Authentication failed, Input {0}@{1} password:\n".format(self.username, self.host_ip)) self.need_password = True self._ssh_fd.connect(hostname=self.host_ip, username=self.username, password=self.password, port=self.ssh_port, disabled_algorithms=self._disabled_rsa_algorithms) except Exception as e: - raise OBDIAGSSHConnException("ssh {0}@{1}: failed, exception:{2}".format(self.host_ip, self.ssh_port, e)) + raise OBDIAGSSHConnException("ssh {0} port {1} failed, exception:{2}".format(self.host_ip, self.ssh_port, e)) else: self._ssh_fd = paramiko.SSHClient() + if remote_client_missing_host_key_policy: + self._ssh_fd.set_missing_host_key_policy(paramiko.MissingHostKeyPolicy()) + else: + self._ssh_fd.load_system_host_keys() self._ssh_fd.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) - self._ssh_fd.load_system_host_keys() self.need_password = True self._ssh_fd.connect(hostname=self.host_ip, username=self.username, password=self.password, port=self.ssh_port, disabled_algorithms=self._disabled_rsa_algorithms) From d3760222bbb96472dc6e467e4d3003df4b4a42ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Wed, 18 Dec 2024 19:32:30 +0800 Subject: [PATCH 4/6] Clean rca old *scene.py files --- rpm/init.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rpm/init.sh b/rpm/init.sh index 5359966..16f993a 100755 --- a/rpm/init.sh +++ b/rpm/init.sh @@ -18,6 +18,10 @@ mkdir -p ${OBDIAG_HOME} && cd ${OBDIAG_HOME} mkdir -p ${OBDIAG_HOME}/check mkdir -p ${OBDIAG_HOME}/log mkdir -p ${OBDIAG_HOME}/display + +# Clean rca old *scene.py files +find "$SOURCE_DIR/rca" -maxdepth 1 -name "*_scene.py" -type f -exec rm -f {} \; + cp -rf ${WORK_DIR}/plugins/* ${OBDIAG_HOME}/ From 5b9c64d7423e42d9630d7c6899a06a818c6907b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Thu, 19 Dec 2024 15:49:49 +0800 Subject: [PATCH 5/6] fix some bugs --- rpm/init.sh | 2 +- src/common/core.py | 4 ---- src/common/diag_cmd.py | 6 ------ src/handler/gather/gather_component_log.py | 6 ++---- 4 files changed, 3 insertions(+), 15 deletions(-) diff --git a/rpm/init.sh b/rpm/init.sh index 16f993a..451a054 100755 --- a/rpm/init.sh +++ b/rpm/init.sh @@ -20,7 +20,7 @@ mkdir -p ${OBDIAG_HOME}/log mkdir -p ${OBDIAG_HOME}/display # Clean rca old *scene.py files -find "$SOURCE_DIR/rca" -maxdepth 1 -name "*_scene.py" -type f -exec rm -f {} \; +find ${OBDIAG_HOME}/rca -maxdepth 1 -name "*_scene.py" -type f -exec rm -f {} \; cp -rf ${WORK_DIR}/plugins/* ${OBDIAG_HOME}/ diff --git a/src/common/core.py b/src/common/core.py index 805b6af..8b1726f 100644 --- a/src/common/core.py +++ b/src/common/core.py @@ -262,7 +262,6 @@ def gather_function(self, function_type, opt): since=Util.get_option(options, 'since'), scope=Util.get_option(options, 'scope'), grep=Util.get_option(options, 'grep'), - encrypt=Util.get_option(options, 'encrypt'), store_dir=Util.get_option(options, 'store_dir'), temp_dir=Util.get_option(options, 'temp_dir'), redact=Util.get_option(options, 'redact'), @@ -304,7 +303,6 @@ def gather_function(self, function_type, opt): since=Util.get_option(options, 'since'), scope=Util.get_option(options, 'scope'), grep=Util.get_option(options, 'grep'), - encrypt=Util.get_option(options, 'encrypt'), store_dir=Util.get_option(options, 'store_dir'), temp_dir=Util.get_option(options, 'temp_dir'), redact=Util.get_option(options, 'redact'), @@ -319,7 +317,6 @@ def gather_function(self, function_type, opt): since=Util.get_option(options, 'since'), scope=Util.get_option(options, 'scope'), grep=Util.get_option(options, 'grep'), - encrypt=Util.get_option(options, 'encrypt'), store_dir=Util.get_option(options, 'store_dir'), temp_dir=Util.get_option(options, 'temp_dir'), redact=Util.get_option(options, 'redact'), @@ -364,7 +361,6 @@ def gather_obproxy_log(self, opt): since=Util.get_option(options, 'since'), scope=Util.get_option(options, 'scope'), grep=Util.get_option(options, 'grep'), - encrypt=Util.get_option(options, 'encrypt'), store_dir=Util.get_option(options, 'store_dir'), temp_dir=Util.get_option(options, 'temp_dir'), redact=Util.get_option(options, 'redact'), diff --git a/src/common/diag_cmd.py b/src/common/diag_cmd.py index 64d96d2..0df9468 100644 --- a/src/common/diag_cmd.py +++ b/src/common/diag_cmd.py @@ -433,9 +433,7 @@ def __init__(self): self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') self.parser.add_option('--temp_dir', type='string', help='the dir for temporarily storing files on nodes', default='/tmp') self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) @@ -459,7 +457,6 @@ def __init__(self): self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') self.parser.add_option('--temp_dir', type='string', help='the dir for temporarily storing files on nodes', default='/tmp') self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) @@ -570,7 +567,6 @@ def __init__(self): self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) self.parser.add_option('--config', action="append", type="string", help='config options Format: --config key=value') @@ -591,7 +587,6 @@ def __init__(self): self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) self.parser.add_option('--config', action="append", type="string", help='config options Format: --config key=value') @@ -655,7 +650,6 @@ def __init__(self): self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') self.parser.add_option('--scope', type='string', help="log type constrains, choices=[obproxy, obproxy_limit, obproxy_stat, obproxy_digest, obproxy_slow, obproxy_diagnosis, obproxy_error, all]", default='all') self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") - self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) self.parser.add_option('--config', action="append", type="string", help='config options Format: --config key=value') diff --git a/src/handler/gather/gather_component_log.py b/src/handler/gather/gather_component_log.py index 7e08f10..edd1e5c 100644 --- a/src/handler/gather/gather_component_log.py +++ b/src/handler/gather/gather_component_log.py @@ -61,7 +61,6 @@ def __init__(self, *args, **kwargs): self.since_option = None self.scope = None self.grep = None - self.encrypt = None self.store_dir = None self.temp_dir = None self.redact = None @@ -83,7 +82,6 @@ def init(self, context, *args, **kwargs): self.since_option = kwargs.get('since', None) self.scope = kwargs.get('scope', None) self.grep = kwargs.get('grep', None) - self.encrypt = kwargs.get('encrypt', None) self.store_dir = kwargs.get('store_dir', None) self.temp_dir = kwargs.get('temp_dir', None) self.redact = kwargs.get('redact', None) @@ -99,7 +97,6 @@ def init(self, context, *args, **kwargs): "tmp_dir": const.GATHER_LOG_TEMPORARY_DIR_DEFAULT, "scope": self.scope, "grep": self.grep, - "encrypt": self.encrypt, "store_dir": self.store_dir, "from_time": self.from_time_str, "to_time": self.to_time_str, @@ -126,7 +123,8 @@ def __check_option(self): # check store_dir if not os.path.exists(self.store_dir): - raise Exception("store_dir: {0} is not exist".format(self.store_dir)) + self.stdio.warn('args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(self.store_dir))) + os.makedirs(os.path.abspath(self.store_dir)) if self.is_scene is False: target_dir = os.path.join("obdiag_gather_pack_{0}".format(TimeUtils.timestamp_to_filename_time(TimeUtils.get_current_us_timestamp()))) self.store_dir = os.path.join(self.store_dir or "./", target_dir) From fcc41048ec5f467fd08129ed98a544c0b278326d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Thu, 19 Dec 2024 17:26:00 +0800 Subject: [PATCH 6/6] fix: gather all not support scope option --- src/common/core.py | 2 -- src/handler/gather/gather_component_log.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/core.py b/src/common/core.py index 8b1726f..52c43c9 100644 --- a/src/common/core.py +++ b/src/common/core.py @@ -301,7 +301,6 @@ def gather_function(self, function_type, opt): from_option=Util.get_option(options, 'from'), to_option=Util.get_option(options, 'to'), since=Util.get_option(options, 'since'), - scope=Util.get_option(options, 'scope'), grep=Util.get_option(options, 'grep'), store_dir=Util.get_option(options, 'store_dir'), temp_dir=Util.get_option(options, 'temp_dir'), @@ -315,7 +314,6 @@ def gather_function(self, function_type, opt): from_option=Util.get_option(options, 'from'), to_option=Util.get_option(options, 'to'), since=Util.get_option(options, 'since'), - scope=Util.get_option(options, 'scope'), grep=Util.get_option(options, 'grep'), store_dir=Util.get_option(options, 'store_dir'), temp_dir=Util.get_option(options, 'temp_dir'), diff --git a/src/handler/gather/gather_component_log.py b/src/handler/gather/gather_component_log.py index edd1e5c..a344741 100644 --- a/src/handler/gather/gather_component_log.py +++ b/src/handler/gather/gather_component_log.py @@ -81,6 +81,8 @@ def init(self, context, *args, **kwargs): self.to_option = self.to_option.strip() self.since_option = kwargs.get('since', None) self.scope = kwargs.get('scope', None) + if isinstance(self.scope, bool): + self.scope = "all" self.grep = kwargs.get('grep', None) self.store_dir = kwargs.get('store_dir', None) self.temp_dir = kwargs.get('temp_dir', None)