From 40683e58836ee9c5352371ab1a7c6725f507eca0 Mon Sep 17 00:00:00 2001 From: Teingi Date: Wed, 3 Jul 2024 17:52:17 +0800 Subject: [PATCH] fix --- README-CN.md | 2 +- README.md | 2 +- common/command.py | 303 +++---- common/config_helper.py | 6 +- common/scene.py | 46 +- common/ssh.py | 277 ------ common/ssh_client/__init__.py | 17 + common/ssh_client/base.py | 80 ++ common/ssh_client/docker_client.py | 82 ++ common/ssh_client/kubernetes_client.py | 112 +++ common/ssh_client/local_client.py | 71 ++ common/ssh_client/remote_client.py | 124 +++ common/ssh_client/ssh.py | 95 ++ common/tool.py | 4 +- conf/inner_config.yml | 7 +- config.py | 16 +- core.py | 3 +- dev_init.sh | 8 +- diag_cmd.py | 831 ++++++++++++++++++ example/operator.yml | 37 + handler/analyzer/analyze_flt_trace.py | 26 +- handler/analyzer/analyze_log.py | 125 ++- handler/analyzer/analyze_sql.py | 48 +- handler/analyzer/analyze_sql_review.py | 36 +- handler/checker/check_handler.py | 11 +- handler/checker/result/verify.py | 2 +- handler/checker/step/data_size.py | 10 +- handler/checker/step/get_system_parameter.py | 15 +- handler/checker/step/ssh.py | 10 +- handler/checker/step/stepbase.py | 8 +- .../observer/cluster/data_path_settings.yaml | 8 - .../column_storage/tenant_parameters.yaml | 7 - .../observer/system/dependent_software.yaml | 7 - .../tasks/observer/system/getenforce.yaml | 17 + .../observer/system/parameter_tcp_wmem.yaml | 2 +- handler/gather/gather_ash_report.py | 13 +- handler/gather/gather_log.py | 115 ++- handler/gather/gather_obadmin.py | 56 +- handler/gather/gather_obproxy_log.py | 62 +- handler/gather/gather_obstack2.py | 179 ++-- handler/gather/gather_perf.py | 52 +- handler/gather/gather_scenes.py | 4 +- handler/gather/gather_sysstat.py | 106 ++- handler/gather/scenes/cpu_high.py | 7 +- handler/gather/scenes/px_collect_log.py | 9 +- handler/gather/step/base.py | 5 +- handler/gather/step/ssh.py | 10 +- handler/meta/sql_meta.py | 40 +- handler/rca/plugins/gather.py | 1 - handler/rca/rca_handler.py | 35 +- handler/rca/scene/disconnection_scene.py | 4 +- handler/rca/scene/major_hold_scene.py | 54 +- init.sh | 2 +- main.py | 2 +- requirements3.txt | 6 +- rpm/build.sh | 2 +- rpm/obdiag-build.sh | 2 +- 57 files changed, 2116 insertions(+), 1105 deletions(-) create mode 100644 common/ssh_client/__init__.py create mode 100644 common/ssh_client/base.py create mode 100644 common/ssh_client/docker_client.py create mode 100644 common/ssh_client/kubernetes_client.py create mode 100644 common/ssh_client/local_client.py create mode 100644 common/ssh_client/remote_client.py create mode 100644 common/ssh_client/ssh.py mode change 100644 => 100755 dev_init.sh create mode 100644 diag_cmd.py create mode 100644 example/operator.yml create mode 100644 handler/checker/tasks/observer/system/getenforce.yaml diff --git a/README-CN.md b/README-CN.md index 6fd9ca6b..30cf2144 100644 --- a/README-CN.md +++ b/README-CN.md @@ -45,7 +45,7 @@ sh /usr/local/oceanbase-diagnostic-tool/init.sh ```shell pip3 install -r requirements3.txt -sh dev_init.sh +./dev_init.sh source ~/.bashrc ``` diff --git a/README.md b/README.md index 02c7a5c4..4a079434 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ To install obdiag on Python >= 3.8, run these commands: ```shell pip3 install -r requirements3.txt -sh dev_init.sh +./dev_init.sh source ~/.bashrc ``` diff --git a/common/command.py b/common/command.py index d418b19f..a3770e43 100644 --- a/common/command.py +++ b/common/command.py @@ -17,9 +17,9 @@ """ import re import subprocess - -from paramiko import SSHException +import traceback from common.ob_connector import OBConnector +from common.ssh_client.ssh import SshClient from common.tool import TimeUtils @@ -30,7 +30,7 @@ def __init__(self, stdio=None): def run(self, cmd): try: self.stdio.verbose("[local host] run cmd = [{0}] on localhost".format(cmd)) - out = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True) + out = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True, executable='/bin/bash') stdout, stderr = out.communicate() if stderr: self.stdio.error("run cmd = [{0}] on localhost, stderr=[{1}]".format(cmd, stderr)) @@ -41,141 +41,115 @@ def run(self, cmd): def run_get_stderr(self, cmd): try: self.stdio.verbose("run cmd = [{0}] on localhost".format(cmd)) - out = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True) + out = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True, executable='/bin/bash') stdout, stderr = out.communicate() return stderr except: self.stdio.error("run cmd = [{0}] on localhost".format(cmd)) -class SshClient(object): - def __init__(self, stdio=None): - self.stdio = stdio - - def run(self, ssh_helper, cmd): - try: - self.stdio.verbose("[remote host {0}] excute cmd = [{1}]".format(ssh_helper.get_name(), cmd)) - stdout = ssh_helper.ssh_exec_cmd(cmd) - self.stdio.verbose("[remote host {0}] excute cmd = [{1}] complete, stdout=[{2}]".format(ssh_helper.get_name(), cmd, stdout)) - return stdout - except Exception as e: - self.stdio.error("[remote host {0}] excute cmd = [{1}] except: [{2}]".format(ssh_helper.get_name(), cmd, e)) - - def run_get_stderr(self, ssh_helper, cmd): - try: - self.stdio.verbose("[remote host {0}] run cmd = [{1}] start ...".format(ssh_helper.get_name(), cmd)) - std = ssh_helper.ssh_exec_cmd_get_stderr(cmd) - return std - except Exception as e: - self.stdio.error("[remote host {0}] run ssh cmd = [{1}] except: {2}".format(ssh_helper.get_name(), cmd, e)) - - def run_ignore_err(self, ssh_helper, cmd): - try: - self.stdio.verbose("[remote host {0}] run cmd = [{1}] start ...".format(ssh_helper.get_name(), cmd)) - std = ssh_helper.ssh_exec_cmd_ignore_err(cmd) - return std - except SSHException as e: - self.stdio.error("[remote host {0}] run ssh cmd = [{1}] except: {2}".format(ssh_helper.get_name(), cmd, e)) - - -def download_file(is_ssh, ssh_helper, remote_path, local_path, stdio=None): +# +# class SshClient(object): +# def __init__(self, stdio=None): +# self.stdio = stdio +# +# def run(self, ssh_helper, cmd): +# try: +# self.stdio.verbose("[remote host {0}] excute cmd = [{1}]".format(ssh_helper.get_name(), cmd)) +# stdout = ssh_helper.ssh_exec_cmd(cmd) +# self.stdio.verbose("[remote host {0}] excute cmd = [{1}] complete, stdout=[{2}]".format(ssh_helper.get_name(), cmd, stdout)) +# return stdout +# except Exception as e: +# self.stdio.error("[remote host {0}] excute cmd = [{1}] except: [{2}]".format(ssh_helper.get_name(), cmd, e)) +# +# def run_get_stderr(self, ssh_helper, cmd): +# try: +# self.stdio.verbose("[remote host {0}] run cmd = [{1}] start ...".format(ssh_helper.get_name(), cmd)) +# std = ssh_helper.ssh_exec_cmd_get_stderr(cmd) +# return std +# except Exception as e: +# self.stdio.error("[remote host {0}] run ssh cmd = [{1}] except: {2}".format(ssh_helper.get_name(), cmd, e)) +# +# def run_ignore_err(self, ssh_helper, cmd): +# try: +# self.stdio.verbose("[remote host {0}] run cmd = [{1}] start ...".format(ssh_helper.get_name(), cmd)) +# std = ssh_helper.ssh_exec_cmd_ignore_err(cmd) +# return std +# except SSHException as e: +# self.stdio.error("[remote host {0}] run ssh cmd = [{1}] except: {2}".format(ssh_helper.get_name(), cmd, e)) + + +def download_file(ssh_client, remote_path, local_path, stdio=None): """ download file - :param args: is_ssh, ssh helper, file path :return: local path """ try: - if is_ssh: - stdio.verbose("Please wait a moment, download file [{0}] from server {1} to [{2}]".format(remote_path, ssh_helper.get_name(), local_path)) - ssh_helper.download(remote_path, local_path) - else: - cmd = "cp -r {0} {1}".format(remote_path, local_path) - LocalClient(stdio).run(cmd) + ssh_client.download(remote_path, local_path) except Exception as e: stdio.error("Download File Failed error: {0}".format(e)) + stdio.verbose(traceback.format_exc()) return local_path -def upload_file(is_ssh, ssh_helper, local_path, remote_path, stdio=None): +def upload_file(ssh_client, local_path, remote_path, stdio=None): """ upload file - :param args: is_ssh, ssh helper, local file path, remote file path :return: local path """ - stdio.verbose("Please wait a moment, upload file to server {0}, local file path {1}, remote file path {2}".format(ssh_helper.get_name(), local_path, remote_path)) + stdio.verbose("Please wait a moment, upload file to server {0}, local file path {1}, remote file path {2}".format(ssh_client.get_name(), local_path, remote_path)) try: - if is_ssh: - ssh_helper.upload(local_path, remote_path) - else: - cmd = "cp -r {0} {1}".format(local_path, remote_path) - LocalClient(stdio).run(cmd) + ssh_client.upload(local_path, remote_path) except Exception as e: stdio.error("Upload File Failed error: {0}".format(e)) -def rm_rf_file(is_ssh, ssh_helper, dir, stdio=None): +def rm_rf_file(ssh_client, dir, stdio=None): """ delete file - :param args: is_ssh, ssh helper, gather log full path :return: """ cmd = "rm -rf {0}".format(dir) - if is_ssh: - SshClient(stdio).run(ssh_helper, cmd) - else: - LocalClient(stdio).run(cmd) + ssh_client.exec_cmd(cmd) -def delete_file_in_folder(is_ssh, ssh_helper, file_path, stdio): +def delete_file_in_folder(ssh_client, file_path, stdio=None): """ delete file - :param args: is_ssh, ssh helper, file_name :return: """ if (file_path is None) or (not 'gather_pack' in file_path): raise Exception("Please check file path, {0}".format(file_path)) cmd = "rm -rf {file_path}/*".format(file_path=file_path) - if is_ssh: - SshClient(stdio).run(ssh_helper, cmd) - else: - LocalClient(stdio).run(cmd) + ssh_client.exec_cmd(cmd) -def is_empty_dir(is_ssh, ssh_helper, dir, stdio=None): +def is_empty_dir(ssh_client, dir, stdio=None): """ determine whether it is an empty folder - :param args: is_ssh, ssh helper, gather log full path :return: true or false """ cmd = "ls -A {gather_path}|wc -w".format(gather_path=dir) - if is_ssh: - file_num = SshClient(stdio).run(ssh_helper, cmd) - else: - file_num = LocalClient(stdio).run(cmd) + file_num = ssh_client.exec_cmd(cmd) if int(file_num) == 0: return True else: return False -def get_file_start_time(is_ssh, ssh_helper, file_name, dir, stdio=None): +def get_file_start_time(ssh_client, file_name, dir, stdio=None): """ get log file start time - :param args: is_ssh, ssh helper, gather log full path :return: true or false """ cmd = "head -n 1 {0}/{1}".format(dir, file_name) - if is_ssh: - first_line_text = SshClient(stdio).run(ssh_helper, cmd) - else: - first_line_text = LocalClient(stdio).run(cmd) + first_line_text = ssh_client.exec_cmd(cmd) return TimeUtils.extract_time_from_log_file_text(str(first_line_text)) -def get_logfile_name_list(is_ssh, ssh_helper, from_time_str, to_time_str, log_dir, log_files, stdio=None): +def get_logfile_name_list(ssh_client, from_time_str, to_time_str, log_dir, log_files, stdio=None): """ get log name list - :param args: is_ssh, ssh helper, from time, to time, log dir, log file list :return: true or false """ stdio.verbose("get log file name list, from time {0}, to time {1}, log dir {2}, log files {3}".format(from_time_str, to_time_str, log_dir, log_files)) @@ -195,7 +169,7 @@ def get_logfile_name_list(is_ssh, ssh_helper, from_time_str, to_time_str, log_di file_start_time_str = "" file_end_time_str = "" elif last_file_dict["prefix_file_name"] == "": - file_start_time_str = get_file_start_time(is_ssh, ssh_helper, file_name, log_dir, stdio) + file_start_time_str = get_file_start_time(ssh_client, file_name, log_dir, stdio) # When two time intervals overlap, need to add the file if (file_end_time_str != "") and (file_start_time_str != "") and (file_start_time_str <= to_time_str) and (file_end_time_str >= from_time_str): log_name_list.append(file_name) @@ -203,17 +177,12 @@ def get_logfile_name_list(is_ssh, ssh_helper, from_time_str, to_time_str, log_di elif file_name.endswith("log") or file_name.endswith("wf"): # Get the first and last lines of text of the file. Here, use a command get_first_line_cmd = "head -n 1 {0}/{1} && tail -n 1 {0}/{1}".format(log_dir, file_name) - if is_ssh: - first_and_last_line_text = SshClient(stdio).run(ssh_helper, get_first_line_cmd) - else: - first_and_last_line_text = LocalClient(stdio).run(get_first_line_cmd) - + first_and_last_line_text = ssh_client.exec_cmd(get_first_line_cmd) # Split the first and last lines of text first_and_last_line_text_list = str(first_and_last_line_text).splitlines() if len(first_and_last_line_text_list) >= 2: first_line_text = first_and_last_line_text_list[0] last_line_text = first_and_last_line_text_list[-1] - # Time to parse the first and last lines of text file_start_time_str = TimeUtils.extract_time_from_log_file_text(first_line_text, stdio) file_end_time = TimeUtils.extract_time_from_log_file_text(last_line_text, stdio) @@ -222,100 +191,99 @@ def get_logfile_name_list(is_ssh, ssh_helper, from_time_str, to_time_str, log_di if (file_start_time_str <= to_time_str) and (file_end_time >= from_time_str): log_name_list.append(file_name) if len(log_name_list) > 0: - stdio.verbose("Find the qualified log file {0} on Server [{1}], " "wait for the next step".format(log_name_list, "localhost" if not is_ssh else ssh_helper.get_name())) + stdio.verbose("Find the qualified log file {0} on Server [{1}], " "wait for the next step".format(log_name_list, ssh_client.get_name())) else: - stdio.warn("No found the qualified log file on Server [{0}]".format("localhost" if not is_ssh else ssh_helper.get_name())) + stdio.warn("No found the qualified log file on Server [{0}]".format(ssh_client.get_name())) return log_name_list -def mkdir(is_ssh, ssh_helper, dir, stdio=None): +def mkdir(ssh_client, dir, stdio=None): """ Create a folder when it does not exist - :param args: is_ssh, ssh helper, folder path :return: """ cmd = "mkdir -p {0}".format(dir) - if is_ssh: - SshClient(stdio).run(ssh_helper, cmd) - else: - LocalClient(stdio).run(cmd) + ssh_client.exec_cmd(cmd) -def delete_empty_file(is_ssh, ssh_helper, dir, stdio=None): +def delete_empty_file(ssh_client, dir): """ delete empty folder - :param args: is_ssh, ssh helper, gather log full path :return: """ cmd = "find {dir} -name '*' -type f -size 0c | xargs -n 1 rm -f".format(dir=dir) - if is_ssh: - SshClient(stdio).run(ssh_helper, cmd) - else: - LocalClient(stdio).run(cmd) + ssh_client.exec_cmd(cmd) -def zip_dir(is_ssh, ssh_helper, father_dir, zip_dir, stdio=None): +def zip_dir(ssh_client, father_dir, zip_dir, stdio=None): """ Compress files through zip - :param args: is_ssh, ssh helper, father dir, zip dir :return: """ cmd = "cd {father_dir} && zip {zip_dir}.zip -rm {zip_dir}".format(father_dir=father_dir, zip_dir=zip_dir) - stdio.verbose("Please wait a moment ...") - if is_ssh: - SshClient(stdio).run(ssh_helper, cmd) - else: - LocalClient(stdio).run(cmd) + ssh_client.exec_cmd(cmd) -def zip_encrypt_dir(is_ssh, ssh_helper, zip_password, father_dir, zip_dir, stdio=None): +def zip_encrypt_dir(ssh_client, zip_password, father_dir, zip_dir, stdio=None): """ Compress files by encryption - :param args: is_ssh, ssh helper, password, raw_log_dir, gather dir name :return: """ cmd = "cd {father_dir} && zip --password {zip_password} {zip_dir}.zip -rm {zip_dir}".format(zip_password=zip_password, father_dir=father_dir, zip_dir=zip_dir) - stdio.verbose("Please wait a moment ...") - if is_ssh: - SshClient(stdio).run(ssh_helper, cmd) - else: - LocalClient(stdio).run(cmd) + ssh_client.exec_cmd(cmd) -def is_support_arch(is_ssh, ssh_helper, stdio=None): +def is_support_arch(ssh_client): """ Determine if it is a supported operating system - :param args: is_ssh, ssh helper :return: """ support_arch_list = ["i386", "i486", "i586", "alpha", "x86_64"] cmd = "arch" try: - if is_ssh: - arch_info = SshClient(stdio).run(ssh_helper, cmd) - else: - arch_info = LocalClient(stdio).run(cmd) + arch_info = ssh_client.exec_cmd(cmd) if arch_info.replace("\n", "") in support_arch_list: return True else: return False except: - stdio.error("get server arch info failed") return False -def get_observer_version(is_ssh, ssh_helper, ob_install_dir, stdio): +def get_observer_version(context): + """ + get observer version + """ + stdio = context.stdio + observer_version = "" + try: + stdio.verbose("get observer version, by sql") + obcluster = context.cluster_config + # by sql + observer_version = get_observer_version_by_sql(obcluster, stdio) + except Exception as e: + try: + stdio.verbose("get observer version, by sql fail. by ssh") + nodes = context.cluster_config.get("servers") + if len(nodes) > 0: + sshclient = SshClient(context, nodes[0]) + ob_install_dir = nodes[0].get("home_path") + observer_version = get_observer_version_by_ssh(sshclient, ob_install_dir, stdio) + except Exception as e: + raise Exception("get observer version fail.") + if observer_version == "": + raise Exception("get observer version fail.") + return observer_version + + +def get_observer_version_by_ssh(ssh_client, ob_install_dir, stdio): """ get observer version - :param args: is_ssh, ssh helper, ob install dir :return: """ ob_version = "" cmd = "{ob_install_dir}/bin/observer --version".format(ob_install_dir=ob_install_dir) - if is_ssh: - ob_version_info = SshClient(stdio).run_get_stderr(ssh_helper, cmd) - else: - ob_version_info = LocalClient(stdio).run_get_stderr(cmd) + ob_version_info = ssh_client.exec_cmd(cmd) stdio.verbose("get observer version, run cmd = [{0}] ".format(cmd)) if ob_version_info is not None: ob_version = re.findall(r'[(]OceanBase.(.+?)[)]', ob_version_info) @@ -324,10 +292,7 @@ def get_observer_version(is_ssh, ssh_helper, ob_install_dir, stdio): return result.strip() else: cmd = "export LD_LIBRARY_PATH={ob_install_dir}/lib && {ob_install_dir}/bin/observer --version".format(ob_install_dir=ob_install_dir) - if is_ssh: - ob_version_info = SshClient(stdio).run_get_stderr(ssh_helper, cmd) - else: - ob_version_info = LocalClient(stdio).run_get_stderr(cmd) + ob_version_info = ssh_client.exec_cmd(cmd) stdio.verbose("get observer version with LD_LIBRARY_PATH,cmd:{0}".format(cmd)) if "REVISION" not in ob_version_info: raise Exception("Please check conf about observer,{0}".format(ob_version_info)) @@ -336,18 +301,20 @@ def get_observer_version(is_ssh, ssh_helper, ob_install_dir, stdio): return result.strip() -def get_obproxy_version(is_ssh, ssh_helper, obproxy_install_dir, stdio): +def get_obproxy_version(context): """ get obproxy version - :param args: is_ssh, ssh helper, ob install dir :return: """ obproxy_version = "" + stdio = context.stdio + obproxy_nodes = context.obproxy_config.get("servers") + if len(obproxy_nodes) < 1: + raise Exception("obproxy_nodes is not exist. get_obproxy_version fail") + obproxy_install_dir = obproxy_nodes[0]["home_path"] cmd = "{obproxy_install_dir}/bin/obproxy --version".format(obproxy_install_dir=obproxy_install_dir) - if is_ssh: - obproxy_version_info = SshClient(stdio).run_get_stderr(ssh_helper, cmd) - else: - obproxy_version_info = LocalClient(stdio).run_get_stderr(cmd) + ssh_client = SshClient(context, obproxy_nodes[0]) + obproxy_version_info = ssh_client.exec_cmd(cmd) stdio.verbose("get obproxy version, run cmd = [{0}] ".format(cmd)) if obproxy_version_info is not None: ob_version = re.findall(r'[(]OceanBase.(.+? +?)[)]', obproxy_version_info) @@ -355,10 +322,7 @@ def get_obproxy_version(is_ssh, ssh_helper, obproxy_install_dir, stdio): return ob_version[0] else: cmd = "export LD_LIBRARY_PATH={obproxy_install_dir}/lib && {obproxy_install_dir}/bin/obproxy --version".format(obproxy_install_dir=obproxy_install_dir) - if is_ssh: - obproxy_version_info = SshClient(stdio).run_get_stderr(ssh_helper, cmd) - else: - obproxy_version_info = LocalClient(stdio).run_get_stderr(cmd) + obproxy_version_info = ssh_client.exec_cmd(cmd) stdio.verbose("get obproxy version with LD_LIBRARY_PATH,cmd:{0}, result:{1}".format(cmd, obproxy_version_info)) if "REVISION" not in obproxy_version_info: raise Exception("Please check conf about proxy,{0}".format(obproxy_version_info)) @@ -385,7 +349,6 @@ def get_observer_version_by_sql(ob_cluster, stdio=None): ob_version_info = ob_connector.execute_sql("select version();") except Exception as e: raise Exception("get_observer_version_by_sql Exception. Maybe cluster'info is error: " + e.__str__()) - ob_version = ob_version_info[0] stdio.verbose("get_observer_version_by_sql ob_version_info is {0}".format(ob_version)) version = re.findall(r'OceanBase(_)?(.CE)?-v(.+)', ob_version[0]) @@ -396,18 +359,14 @@ def get_observer_version_by_sql(ob_cluster, stdio=None): return version[0] -def get_observer_pid(is_ssh, ssh_helper, ob_install_dir, stdio=None): +def get_observer_pid(ssh_client, ob_install_dir, stdio=None): """ get observer pid - :param args: is_ssh, ssh helper, ob install dir :return: """ try: cmd = "cat {ob_install_dir}/run/observer.pid".format(ob_install_dir=ob_install_dir) - if is_ssh: - pids = SshClient(stdio).run(ssh_helper, cmd) - else: - pids = LocalClient(stdio).run(cmd) + pids = ssh_client.exec_cmd(cmd) pid_list = pids.split() stdio.verbose("get observer pid, run cmd = [{0}], result:{1} ".format(cmd, pid_list)) except: @@ -416,83 +375,61 @@ def get_observer_pid(is_ssh, ssh_helper, ob_install_dir, stdio=None): return pid_list -def delete_file_force(is_ssh, ssh_helper, file_name, stdio=None): +def delete_file_force(ssh_client, file_name, stdio=None): """ delete file force - :param args: is_ssh, ssh helper, file_name :return: """ cmd = "rm -rf {0}".format(file_name) - if is_ssh: - SshClient(stdio).run(ssh_helper, cmd) - else: - LocalClient(stdio).run(cmd) + ssh_client.exec_cmd(cmd) -def delete_empty_file(is_ssh, ssh_helper, file_path, stdio=None): +def delete_empty_file(ssh_client, file_path, stdio=None): """ delete empty file - :param args: is_ssh, ssh helper, file_name :return: """ cmd = "find {file_path} -name '*' -type f -size 0c | xargs -n 1 rm -f".format(file_path=file_path) - if is_ssh: - SshClient(stdio).run(ssh_helper, cmd) - else: - LocalClient(stdio).run(cmd) + ssh_client.exec_cmd(cmd) -def delete_file(is_ssh, ssh_helper, file_path, stdio=None): +def delete_file(ssh_client, file_path, stdio=None): """ delete file - :param args: is_ssh, ssh helper, file_name :return: """ cmd = "rm -rf {file_path}".format(file_path=file_path) - if is_ssh: - SshClient(stdio).run(ssh_helper, cmd) - else: - LocalClient(stdio).run(cmd) + ssh_client.exec_cmd(cmd) -def get_file_size(is_ssh, ssh_helper, file_path, stdio=None): +def get_file_size(ssh_client, file_path, stdio=None): """ get file size - :param args: is_ssh, ssh helper, file_path :return: """ cmd = "ls -nl %s | awk '{print $5}'" % file_path - if is_ssh: - file_size = SshClient(stdio).run(ssh_helper, cmd) - else: - file_size = LocalClient(stdio).run(cmd) - return file_size + return ssh_client.exec_cmd(cmd) -def is_empty_dir(is_ssh, ssh_helper, dir_path, stdio=None): +def is_empty_dir(ssh_client, dir_path, stdio=None): """ is empty dir - :param args: is_ssh, ssh helper, dir_path :return: True or False """ cmd = "ls -A {dir_path}|wc -w".format(dir_path=dir_path) - if is_ssh: - file_num = SshClient(stdio).run(ssh_helper, cmd) - else: - file_num = LocalClient(stdio).run(cmd) + file_num = ssh_client.exec_cmd(cmd) if int(file_num) == 0: return True else: return False -def is_empty_file(is_ssh, ssh_helper, file_path, stdio=None): +def is_empty_file(ssh_client, file_path, stdio=None): """ is empty file - :param args: is_ssh, ssh helper, file_path :return: True or False """ - file_size = get_file_size(is_ssh, ssh_helper, file_path, stdio) + file_size = get_file_size(ssh_client, file_path, stdio) if int(file_size) == 0: return True else: @@ -535,13 +472,9 @@ def analyze_log_get_sqc_addr(uzip_dir, stdio): return None -def find_home_path_by_port(is_ssh, ssh_helper, internal_port_str, stdio): +def find_home_path_by_port(ssh_client, internal_port_str, stdio): cmd = "ps aux | grep observer | grep 'P {internal_port_str}' | grep -oP '/[^\s]*/bin/observer' ".format(internal_port_str=internal_port_str) - if is_ssh: - stdout = SshClient(stdio).run(ssh_helper, cmd) - else: - stdout = LocalClient(stdio).run(cmd) - + stdout = ssh_client.exec_cmd(cmd) str_list = stdout.strip().split('\n') home_path = "" for original_str in str_list: diff --git a/common/config_helper.py b/common/config_helper.py index 979d59b9..f46433a6 100644 --- a/common/config_helper.py +++ b/common/config_helper.py @@ -20,7 +20,7 @@ import time from collections import OrderedDict -from common.command import get_observer_version_by_sql +from common.command import get_observer_version from common.constant import const from common.ob_connector import OBConnector from common.tool import DirectoryUtil @@ -51,7 +51,7 @@ def __init__(self, context): } def get_cluster_name(self): - ob_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) + ob_version = get_observer_version(self.context) obConnetcor = OBConnector(ip=self.db_host, port=self.db_port, username=self.sys_tenant_user, password=self.sys_tenant_password, stdio=self.stdio, timeout=100) if ob_version.startswith("3") or ob_version.startswith("2"): sql = "select cluster_name from oceanbase.v$ob_cluster" @@ -64,7 +64,7 @@ def get_cluster_name(self): return "obcluster" def get_host_info_list_by_cluster(self): - ob_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) + ob_version = get_observer_version(self.context) obConnetcor = OBConnector(ip=self.db_host, port=self.db_port, username=self.sys_tenant_user, password=self.sys_tenant_password, stdio=self.stdio, timeout=100) sql = "select SVR_IP, SVR_PORT, ZONE, BUILD_VERSION from oceanbase.DBA_OB_SERVERS" if ob_version.startswith("3") or ob_version.startswith("2") or ob_version.startswith("1"): diff --git a/common/scene.py b/common/scene.py index 9795ef5a..c9378923 100644 --- a/common/scene.py +++ b/common/scene.py @@ -15,9 +15,8 @@ @file: scene.py @desc: """ -from common.ssh import SshHelper from common.tool import StringUtils -from common.command import get_observer_version, get_obproxy_version, get_observer_version_by_sql +from common.command import get_observer_version, get_obproxy_version def filter_by_version(scene, cluster, stdio=None): @@ -66,44 +65,21 @@ def filter_by_version(scene, cluster, stdio=None): raise Exception("filter_by_version Exception : {0}".format(e)) -def get_version(nodes, type, cluster, stdio=None): +def get_version_by_type(context, type, stdio=None): try: - if len(nodes) < 1: - raise Exception("input nodes is empty, please check your config") - node = nodes[0] - ssh = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) + stdio = context.stdio version = "" - if type == "observer": + if type == "observer" or type == "other": try: - version = get_observer_version_by_sql(cluster, stdio) + version = get_observer_version(context) except Exception as e: - stdio.warn("get observer version by sql fail, use node ssher to get. Exception:{0}".format(e)) - version = get_observer_version(True, ssh, nodes[0]["home_path"], stdio) - elif type == "obproxy": - version = get_obproxy_version(True, ssh, nodes[0]["home_path"], stdio) - return version - except Exception as e: - stdio.exception("can't get version, Exception: {0}".format(e)) - raise Exception("can't get version, Exception: {0}".format(e)) - - -def get_obproxy_and_ob_version(obproxy_nodes, nodes, type, stdio=None): - try: - if type == "observer" or type == "other": - if len(nodes) < 1: - raise Exception("input nodes is empty, please check your config") - node = nodes[0] - ssh = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) - version = get_observer_version(True, ssh, nodes[0]["home_path"], stdio) + stdio.warn("get observer version fail . Exception:{0}".format(e)) + raise Exception("can't get observer version, Exception: {0}".format(e)) elif type == "obproxy": - if len(nodes) < 1: - raise Exception("input obproxy nodes is empty, please check your config") - node = obproxy_nodes[0] - ssh = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) - version = get_obproxy_version(True, ssh, nodes[0]["home_path"], stdio) + version = get_obproxy_version(context) else: - raise Exception("type is {0} . No func to get the version".format(type)) + raise Exception("type is {0} . No support to get the version".format(type)) return version except Exception as e: - stdio.exception("can't get version, Exception: {0}".format(e)) - raise Exception("can't get version, Exception: {0}".format(e)) + stdio.exception("can't get {0} version, Exception: {1}".format(type, e)) + raise Exception("can't get {0} version, Exception: {1}".format(type, e)) diff --git a/common/ssh.py b/common/ssh.py index c9f468b7..ccd8a40c 100644 --- a/common/ssh.py +++ b/common/ssh.py @@ -802,280 +802,3 @@ def file_uploader(self, local_dir, remote_dir, stdio=None): except: stdio.exception("") stdio.verbose('Failed to get %s' % remote_dir) - - -# TODO ENV_DISABLE_RSA_ALGORITHMS need get by context.inner_context -ENV_DISABLE_RSA_ALGORITHMS = 0 - - -def dis_rsa_algorithms(state=0): - """ - Disable RSA algorithms in OpenSSH server. - """ - global ENV_DISABLE_RSA_ALGORITHMS - ENV_DISABLE_RSA_ALGORITHMS = state - - -class SshHelper(object): - def __init__(self, is_ssh=None, host_ip=None, username=None, password=None, ssh_port=None, key_file=None, node=None, stdio=None): - if node is None: - node = {} - self.is_ssh = is_ssh - self.stdio = stdio - self.host_ip = host_ip - self.username = username - self.ssh_port = node.get("ssh_port") or ssh_port - self.need_password = True - self.password = node.get("ssh_password") or password - self.key_file = node.get("ssh_key_file") or key_file - self.key_file = os.path.expanduser(self.key_file) - self.ssh_type = node.get("ssh_type") or "remote" - self._ssh_fd = None - self._sftp_client = None - if "ssh_type" in node and node.get("ssh_type") == "docker": - try: - self.ssh_type = node["ssh_type"] - self.stdio.verbose("use ssh_type:{0} , node info : {1}".format(self.ssh_type, StringUtils.node_cut_passwd_for_log(node))) - self.node = node - # docker_permissions_check - if self.ssh_type == "docker": - self.client = docker.from_env() - if "container_name" not in node: - self.stdio.error("SshHelper init docker Exception: 'container_name' not in node") - raise Exception("SshHelper init docker Exception: 'container_name' not in node") - else: - self.stdio.error("SshHelper init not support the ssh_type : {0}".format(self.ssh_type)) - raise Exception("SshHelper init not support the ssh_type : {0}".format(self.ssh_type)) - - except Exception as e: - self.stdio.error("SshHelper init docker Exception: {0}".format(e)) - raise Exception("SshHelper init docker Exception: {0}".format(e)) - - return - - if self.is_ssh: - self._disabled_rsa_algorithms = None - DISABLED_ALGORITHMS = dict(pubkeys=["rsa-sha2-512", "rsa-sha2-256"]) - if ENV_DISABLE_RSA_ALGORITHMS == 1: - self._disabled_rsa_algorithms = DISABLED_ALGORITHMS - self.ssh_type = "remote" - if len(self.key_file) > 0: - try: - self._ssh_fd = paramiko.SSHClient() - self._ssh_fd.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) - self._ssh_fd.load_system_host_keys() - self._ssh_fd.connect(hostname=host_ip, username=username, key_filename=self.key_file, port=ssh_port, disabled_algorithms=self._disabled_rsa_algorithms) - except AuthenticationException: - self.password = input("Authentication failed, Input {0}@{1} password:\n".format(username, host_ip)) - self.need_password = True - self._ssh_fd.connect(hostname=host_ip, username=username, password=password, port=ssh_port, disabled_algorithms=self._disabled_rsa_algorithms) - except Exception as e: - raise OBDIAGSSHConnException("ssh {0}@{1}: failed, exception:{2}".format(username, host_ip, e)) - else: - self._ssh_fd = paramiko.SSHClient() - self._ssh_fd.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) - self._ssh_fd.load_system_host_keys() - self.need_password = True - self._ssh_fd.connect(hostname=host_ip, username=username, password=password, port=ssh_port, disabled_algorithms=self._disabled_rsa_algorithms) - - def ssh_exec_cmd(self, cmd): - if self.ssh_type == "docker": - try: - self.stdio.verbose("ssh_exec_cmd docker {0} cmd: {1}".format(self.node.get("container_name"), cmd)) - client_result = self.client.containers.get(self.node["container_name"]) - result = client_result.exec_run( - cmd=["bash", "-c", cmd], - detach=False, - stdout=True, - stderr=True, - ) - if result.exit_code != 0: - raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.node["container_name"], cmd, result.output.decode('utf-8'))) - - except Exception as e: - self.stdio.error("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) - raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) - - return result.output.decode('utf-8') - try: - stdin, stdout, stderr = self._ssh_fd.exec_command(cmd) - err_text = stderr.read() - if len(err_text): - raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.host_ip, cmd, err_text)) - except SSHException as e: - raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.host_ip, cmd, e)) - return stdout.read().decode('utf-8') - - def ssh_exec_cmd_ignore_err(self, cmd): - if self.ssh_type == "docker": - try: - client_result = self.client.containers.get(self.node["container_name"]) - result = client_result.exec_run( - cmd=["bash", "-c", cmd], - detach=False, - stdout=True, - stderr=True, - ) - except Exception as e: - self.stdio.error("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) - raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) - - return result.output.decode('utf-8') - - try: - stdin, stdout, stderr = self._ssh_fd.exec_command(cmd) - return stdout.read().decode('utf-8') - except SSHException as e: - print("Execute Shell command on server {0} failed,command=[{1}], exception:{2}".format(self.node, cmd, e)) - - def ssh_exec_cmd_ignore_exception(self, cmd): - if self.ssh_type == "docker": - try: - client_result = self.client.containers.get(self.node["container_name"]) - result = client_result.exec_run( - cmd=["bash", "-c", cmd], - detach=False, - stdout=True, - stderr=True, - ) - return result.output.decode('utf-8') - except Exception as e: - self.stdio.error("sshHelper ssh_exec_cmd_ignore_exception docker Exception: {0}".format(e)) - pass - # raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) - return - - try: - stdin, stdout, stderr = self._ssh_fd.exec_command(cmd) - return stderr.read().decode('utf-8') - except SSHException as e: - pass - - def ssh_exec_cmd_get_stderr(self, cmd): - if self.ssh_type == "docker": - try: - client_result = self.client.containers.get(self.node["container_name"]) - result = client_result.exec_run( - cmd=["bash", "-c", cmd], - detach=False, - stdout=True, - stderr=True, - ) - return result.output.decode('utf-8') - except Exception as e: - self.stdio.error("sshHelper ssh_exec_cmd_ignore_exception docker Exception: {0}".format(e)) - pass - # raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) - return - try: - stdin, stdout, stderr = self._ssh_fd.exec_command(cmd) - return stderr.read().decode('utf-8') - except SSHException as e: - pass - - def progress_bar(self, transferred, to_be_transferred, suffix=''): - bar_len = 20 - filled_len = int(round(bar_len * transferred / float(to_be_transferred))) - percents = round(20.0 * transferred / float(to_be_transferred), 1) - bar = '\033[32;1m%s\033[0m' % '=' * filled_len + '-' * (bar_len - filled_len) - print_percents = round((percents * 5), 1) - sys.stdout.flush() - sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) - if transferred == to_be_transferred: - sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) - print() - - def download(self, remote_path, local_path): - if self.ssh_type == "docker": - try: - self.stdio.verbose("remote_path: {0}:{1} to local_path:{2}".format(self.node["container_name"], remote_path, local_path)) - client_result = self.client.containers.get(self.node["container_name"]) - data, stat = client_result.get_archive(remote_path) - with open(local_path, "wb") as f: - for chunk in data: - f.write(chunk) - return - except Exception as e: - self.stdio.error("sshHelper download docker Exception: {0}".format(e)) - raise Exception("sshHelper download docker Exception: {0}".format(e)) - return - - transport = self._ssh_fd.get_transport() - self._sftp_client = paramiko.SFTPClient.from_transport(transport) - print('Download {0}:{1}'.format(self.host_ip, remote_path)) - self._sftp_client.get(remote_path, local_path, callback=self.progress_bar) - self._sftp_client.close() - - def translate_byte(self, B): - B = float(B) - KB = float(1024) - MB = float(KB**2) - GB = float(MB**2) - TB = float(GB**2) - if B < KB: - return '{} {}'.format(B, 'bytes' if B > 1 else "byte") - elif KB < B < MB: - return '{:.2f} KB'.format(B / KB) - elif MB < B < GB: - return '{:.2f} MB'.format(B / MB) - elif GB < B < TB: - return '{:.2f} GB'.format(B / GB) - else: - return '{:.2f} TB'.format(B / TB) - - def upload(self, remote_path, local_path): - if self.ssh_type == "docker": - try: - self.stdio.verbose(" local_path:{0} to remote_path:{1}:{2}".format(local_path, self.node["container_name"], remote_path)) - - self.client.containers.get(self.node["container_name"]).put_archive(remote_path, local_path) - - return - except Exception as e: - self.stdio.error("sshHelper upload docker Exception: {0}".format(e)) - raise Exception("sshHelper upload docker Exception: {0}".format(e)) - return - transport = self._ssh_fd.get_transport() - self._sftp_client = paramiko.SFTPClient.from_transport(transport) - self._sftp_client.put(remote_path, local_path) - self._sftp_client.close() - - def ssh_close(self): - if self.ssh_type == "docker": - self.client.close() - return - if self._sftp_client is not None: - self._sftp_client.close() - self._sftp_client = None - - def __del__(self): - if self._sftp_client is not None: - self._sftp_client.close() - self._sftp_client = None - - def ssh_invoke_shell_switch_user(self, new_user, cmd, time_out): - if self.ssh_type == "docker": - try: - exec_id = self.client.exec_create(container=self.node["container_name"], command=['su', '- ' + new_user]) - response = self.client.exec_start(exec_id) - - return response - except Exception as e: - self.stdio.error("sshHelper ssh_invoke_shell_switch_user docker Exception: {0}".format(e)) - raise Exception("sshHelper ssh_invoke_shell_switch_user docker Exception: {0}".format(e)) - return - try: - ssh = self._ssh_fd.invoke_shell() - ssh.send('su {0}\n'.format(new_user)) - ssh.send('{}\n'.format(cmd)) - time.sleep(time_out) - self._ssh_fd.close() - result = ssh.recv(65535) - except SSHException as e: - raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.host_ip, cmd, e)) - return result - - def get_name(self): - if self.ssh_type == "docker": - return "(docker)" + self.node.get("container_name") - return self.host_ip diff --git a/common/ssh_client/__init__.py b/common/ssh_client/__init__.py new file mode 100644 index 00000000..55bef883 --- /dev/null +++ b/common/ssh_client/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/24 +@file: __init__.py +@desc: +""" diff --git a/common/ssh_client/base.py b/common/ssh_client/base.py new file mode 100644 index 00000000..4ac6deed --- /dev/null +++ b/common/ssh_client/base.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/24 +@file: base.py +@desc: +""" +import sys + +from stdio import SafeStdio + + +class SsherClient(SafeStdio): + def __init__(self, context, node): + super().__init__() + self.context = context + if context is not None: + self.stdio = self.context.stdio + else: + self.stdio = None + self.node = node + self.ssh_type = node.get("ssh_type") or "remote" + self.client = None + + def exec_cmd(self, cmd): + raise Exception("the client type is not support exec_cmd") + + def download(self, remote_path, local_path): + raise Exception("the client type is not support download") + + def upload(self, remote_path, local_path): + raise Exception("the client type is not support upload") + + def ssh_invoke_shell_switch_user(self, new_user, cmd, time_out): + raise Exception("the client type is not support ssh invoke shell switch user") + + def ssh_close(self): + return + + def get_name(self): + return "not defined" + + def get_ip(self): + return self.client.get_ip() + + def progress_bar(self, transferred, to_be_transferred, suffix=''): + bar_len = 20 + filled_len = int(round(bar_len * transferred / float(to_be_transferred))) + percents = round(20.0 * transferred / float(to_be_transferred), 1) + bar = '\033[32;1m%s\033[0m' % '=' * filled_len + '-' * (bar_len - filled_len) + print_percents = round((percents * 5), 1) + sys.stdout.flush() + sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) + if transferred == to_be_transferred: + sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) + print() + + def translate_byte(self, B): + if B < 0: + B = -B + return '-' + self.translate_byte(B) + if B == 0: + return '0B' + units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] + k = 1024 + i = 0 + while B >= k and i < len(units) - 1: + B /= k + i += 1 + return f"{B:.2f} {units[i]}" diff --git a/common/ssh_client/docker_client.py b/common/ssh_client/docker_client.py new file mode 100644 index 00000000..6a74a97a --- /dev/null +++ b/common/ssh_client/docker_client.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/24 +@file: docker_client.py +@desc: +""" +import docker + +from common.obdiag_exception import OBDIAGShellCmdException +from common.ssh_client.base import SsherClient + + +class DockerClient(SsherClient): + def __init__(self, context=None, node=None): + super().__init__(context, node) + self.container_name = self.node.get("container_name") + self.client = docker.from_env() + + def exec_cmd(self, cmd): + try: + self.stdio.verbose("ssh_exec_cmd docker {0} cmd: {1}".format(self.container_name, cmd)) + client_result = self.client.containers.get(self.container_name) + result = client_result.exec_run( + cmd=["bash", "-c", cmd], + detach=False, + stdout=True, + stderr=True, + ) + if result.exit_code != 0: + raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.container_name, cmd, result.output.decode('utf-8'))) + except Exception as e: + self.stdio.error("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) + raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e)) + return result.output.decode('utf-8') + + def download(self, remote_path, local_path): + try: + self.stdio.verbose("remote_path: {0}:{1} to local_path:{2}".format(self.node["container_name"], remote_path, local_path)) + client_result = self.client.containers.get(self.node["container_name"]) + data, stat = client_result.get_archive(remote_path) + with open(local_path, "wb") as f: + for chunk in data: + f.write(chunk) + + except Exception as e: + self.stdio.error("sshHelper download docker Exception: {0}".format(e)) + raise Exception("sshHelper download docker Exception: {0}".format(e)) + + def upload(self, remote_path, local_path): + try: + self.stdio.verbose(" local_path:{0} to remote_path:{1}:{2}".format(local_path, self.node["container_name"], remote_path)) + self.client.containers.get(self.node["container_name"]).put_archive(remote_path, local_path) + return + except Exception as e: + self.stdio.error("sshHelper upload docker Exception: {0}".format(e)) + raise Exception("sshHelper upload docker Exception: {0}".format(e)) + + def ssh_invoke_shell_switch_user(self, new_user, cmd, time_out): + try: + exec_id = self.client.exec_create(container=self.node["container_name"], command=['su', '- ' + new_user]) + response = self.client.exec_start(exec_id) + return response + except Exception as e: + self.stdio.error("sshHelper ssh_invoke_shell_switch_user docker Exception: {0}".format(e)) + raise Exception("sshHelper ssh_invoke_shell_switch_user docker Exception: {0}".format(e)) + + def get_name(self): + return "docker_{0}".format(self.container_name) + + def get_ip(self): + return self.client.containers.get(self.node["container_name"]).attrs['NetworkSettings']['Networks']['bridge']["IPAddress"] diff --git a/common/ssh_client/kubernetes_client.py b/common/ssh_client/kubernetes_client.py new file mode 100644 index 00000000..d6bc6ea9 --- /dev/null +++ b/common/ssh_client/kubernetes_client.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/24 +@file: kubernetes_client.py +@desc: +""" + +from common.ssh_client.base import SsherClient +from kubernetes import client, config +from kubernetes.stream import stream + + +class KubernetesClient(SsherClient): + def __init__(self, context=None, node=None): + super().__init__(context, node) + try: + self.namespace = self.node.get("namespace") + self.pod_name = self.node.get("pod_name") + self.container_name = self.node.get("container_name") or "observer" + config_file = self.node.get("kubernetes_config_file") + if config_file is None or config_file == "": + config.kube_config.load_kube_config() + else: + config.kube_config.load_kube_config(config_file=config_file) + self.client = client.CoreV1Api() + except Exception as e: + raise Exception("KubernetesClient load_kube_config error. Please check the config file. {0}".format(e)) + + def exec_cmd(self, cmd): + exec_command = ['/bin/sh', '-c', cmd] + self.stdio.verbose("KubernetesClient exec_cmd: {0}".format(cmd)) + resp = stream(self.client.connect_get_namespaced_pod_exec, self.pod_name, self.namespace, command=exec_command, stderr=True, stdin=False, stdout=True, tty=False, container=self.container_name) + self.stdio.verbose("KubernetesClient exec_cmd.resp: {0}".format(resp)) + if "init system (PID 1). Can't operate." in resp: + return "KubernetesClient can't get the resp by {0}".format(cmd) + return resp + + def download(self, remote_path, local_path): + return self.__download_file_from_pod(self.namespace, self.pod_name, self.container_name, remote_path, local_path) + + def __download_file_from_pod(self, namespace, pod_name, container_name, file_path, local_path): + exec_command = ['tar', 'cf', '-', '-C', '/', file_path] + resp = stream(self.client.connect_get_namespaced_pod_exec, pod_name, namespace, command=exec_command, stderr=True, stdin=False, stdout=True, tty=False, container=container_name, _preload_content=False) + with open(local_path, 'wb') as file: + while resp.is_open(): + resp.update(timeout=1) + if resp.peek_stdout(): + out = resp.read_stdout() + file.write(out.encode('utf-8')) + if resp.peek_stderr(): + err = resp.read_stderr() + self.stdio.error("ERROR: ", err) + break + resp.close() + + def upload(self, remote_path, local_path): + return self.__upload_file_to_pod(self.namespace, self.pod_name, self.container_name, local_path, remote_path) + + def __upload_file_to_pod(self, namespace, pod_name, container_name, local_path, remote_path): + config.load_kube_config() + v1 = client.CoreV1Api() + exec_command = ['tar', 'xvf', '-', '-C', '/', remote_path] + with open(local_path, 'rb') as file: + resp = stream(v1.connect_get_namespaced_pod_exec, pod_name, namespace, command=exec_command, stderr=True, stdin=True, stdout=True, tty=False, container=container_name, _preload_content=False) + # Support data flow for tar command + commands = [] + commands.append(file.read()) + while resp.is_open(): + resp.update(timeout=1) + if resp.peek_stdout(): + self.stdio.verbose("STDOUT: %s" % resp.read_stdout()) + if resp.peek_stderr(): + self.stdio.error("STDERR: %s" % resp.read_stderr()) + if commands: + c = commands.pop(0) + resp.write_stdin(c) + else: + break + resp.close() + + def ssh_invoke_shell_switch_user(self, new_user, cmd, time_out): + return self.__ssh_invoke_shell_switch_user(new_user, cmd, time_out) + + def __ssh_invoke_shell_switch_user(self, new_user, cmd, time_out): + command = ['/bin/sh', '-c', cmd] + # 构建执行tar命令串，该命令串在切换用户后执行 + exec_command = ['su', '-u', new_user, "&"] + command + resp = stream(self.client.connect_get_namespaced_pod_exec, self.pod_name, self.namespace, command=exec_command, stderr=True, stdin=False, stdout=True, tty=False, container=self.container_name) + parts = resp.split('\n', maxsplit=1) + if len(parts) < 2: + return "" + result = parts[1] + return result + + def get_name(self): + return "kubernetes_{0}_{1}".format(self.namespace, self.pod_name) + + def get_ip(self): + if self.node.get("ip") is None: + raise Exception("kubernetes need set the ip of observer") + return self.node.get("ip") diff --git a/common/ssh_client/local_client.py b/common/ssh_client/local_client.py new file mode 100644 index 00000000..52ee2ca9 --- /dev/null +++ b/common/ssh_client/local_client.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/24 +@file: local_client.py +@desc: +""" + +from common.ssh_client.base import SsherClient +import subprocess32 as subprocess +import shutil + + +class LocalClient(SsherClient): + def __init__(self, context=None, node=None): + super().__init__(context, node) + + def exec_cmd(self, cmd): + try: + self.stdio.verbose("[local host] run cmd = [{0}] on localhost".format(cmd)) + out = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable='/bin/bash') + stdout, stderr = out.communicate() + if stderr: + return stderr.decode('utf-8') + return stdout.decode('utf-8') + except Exception as e: + self.stdio.error("run cmd = [{0}] on localhost, Exception = [{1}]".format(cmd, e)) + raise Exception("[localhost] Execute Shell command failed, command=[{0}] Exception = [{1}]".format(cmd, e)) + + def download(self, remote_path, local_path): + try: + shutil.copy(remote_path, local_path) + except Exception as e: + self.stdio.error("download file from localhost, remote_path=[{0}], local_path=[{1}], error=[{2}]".format(remote_path, local_path, str(e))) + raise Exception("download file from localhost, remote_path=[{0}], local_path=[{1}], error=[{2}]".format(remote_path, local_path, str(e))) + + def upload(self, remote_path, local_path): + try: + shutil.copy(local_path, remote_path) + except Exception as e: + self.stdio.error("upload file to localhost, remote _path =[{0}], local _path=[{1}], error=[{2}]".format(remote_path, local_path, str(e))) + raise Exception("[local] upload file to localhost, remote _path =[{0}], local _path=[{1}], error=[{2}]".format(remote_path, local_path, str(e))) + + def ssh_invoke_shell_switch_user(self, new_user, cmd, time_out): + try: + cmd = "su - {0} -c '{1}'".format(new_user, cmd) + self.stdio.verbose("[local host] ssh_invoke_shell_switch_user cmd = [{0}] on localhost".format(cmd)) + out = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True, executable='/bin/bash') + stdout, stderr = out.communicate() + if stderr: + return stderr.decode('utf-8') + return stdout.decode('utf-8') + except: + self.stdio.error("run cmd = [{0}] on localhost".format(cmd)) + raise Exception("the client type is not support ssh invoke shell switch user") + + def get_name(self): + return "local" + + def get_ip(self): + return self.client.get_ip() diff --git a/common/ssh_client/remote_client.py b/common/ssh_client/remote_client.py new file mode 100644 index 00000000..a24146fd --- /dev/null +++ b/common/ssh_client/remote_client.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/24 +@file: remote_client.py +@desc: +""" + + +import os +import sys +import time +import paramiko +from paramiko.ssh_exception import SSHException, AuthenticationException +from common.obdiag_exception import OBDIAGShellCmdException, OBDIAGSSHConnException +from common.ssh_client.base import SsherClient + +ENV_DISABLE_RSA_ALGORITHMS = 0 + + +def dis_rsa_algorithms(state=0): + """ + Disable RSA algorithms in OpenSSH server. + """ + global ENV_DISABLE_RSA_ALGORITHMS + ENV_DISABLE_RSA_ALGORITHMS = state + + +class RemoteClient(SsherClient): + def __init__(self, context, node): + super().__init__(context, node) + self._sftp_client = None + self._disabled_rsa_algorithms = None + self.host_ip = self.node.get("ip") + self.username = self.node.get("username") + self.ssh_port = self.node.get("ssh_port") + self.need_password = True + self.password = self.node.get("ssh_password") + self.key_file = self.node.get("ssh_key_file") + self.key_file = os.path.expanduser(self.key_file) + self._ssh_fd = None + self._sftp_client = None + DISABLED_ALGORITHMS = dict(pubkeys=["rsa-sha2-512", "rsa-sha2-256"]) + if ENV_DISABLE_RSA_ALGORITHMS == 1: + self._disabled_rsa_algorithms = DISABLED_ALGORITHMS + self.ssh_type = "remote" + if len(self.key_file) > 0: + try: + self._ssh_fd = paramiko.SSHClient() + self._ssh_fd.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) + self._ssh_fd.load_system_host_keys() + self._ssh_fd.connect(hostname=self.host_ip, username=self.username, key_filename=self.key_file, port=self.ssh_port, disabled_algorithms=self._disabled_rsa_algorithms) + except AuthenticationException: + self.password = input("Authentication failed, Input {0}@{1} password:\n".format(self.username, self.ssh_port)) + self.need_password = True + self._ssh_fd.connect(hostname=self.host_ip, username=self.username, password=self.password, port=self.ssh_port, disabled_algorithms=self._disabled_rsa_algorithms) + except Exception as e: + raise OBDIAGSSHConnException("ssh {0}@{1}: failed, exception:{2}".format(self.host_ip, self.ssh_port, e)) + else: + self._ssh_fd = paramiko.SSHClient() + self._ssh_fd.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) + self._ssh_fd.load_system_host_keys() + self.need_password = True + self._ssh_fd.connect(hostname=self.host_ip, username=self.username, password=self.password, port=self.ssh_port, disabled_algorithms=self._disabled_rsa_algorithms) + + def exec_cmd(self, cmd): + try: + stdin, stdout, stderr = self._ssh_fd.exec_command(cmd) + err_text = stderr.read() + if len(err_text): + return err_text.decode('utf-8') + return stdout.read().decode('utf-8') + except SSHException as e: + raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.host_ip, cmd, e)) + + def download(self, remote_path, local_path): + transport = self._ssh_fd.get_transport() + self._sftp_client = paramiko.SFTPClient.from_transport(transport) + self.stdio.verbose('Download {0}:{1}'.format(self.host_ip, remote_path)) + self._sftp_client.get(remote_path, local_path, callback=self.progress_bar) + self._sftp_client.close() + + def progress_bar(self, transferred, to_be_transferred, suffix=''): + bar_len = 20 + filled_len = int(round(bar_len * transferred / float(to_be_transferred))) + percents = round(20.0 * transferred / float(to_be_transferred), 1) + bar = '\033[32;1m%s\033[0m' % '=' * filled_len + '-' * (bar_len - filled_len) + print_percents = round((percents * 5), 1) + sys.stdout.flush() + sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) + if transferred == to_be_transferred: + sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix)) + print() + + def upload(self, remote_path, local_path): + transport = self._ssh_fd.get_transport() + self._sftp_client = paramiko.SFTPClient.from_transport(transport) + self._sftp_client.put(remote_path, local_path) + self._sftp_client.close() + + def ssh_invoke_shell_switch_user(self, new_user, cmd, time_out): + try: + ssh = self._ssh_fd.invoke_shell() + ssh.send('su {0}\n'.format(new_user)) + ssh.send('{}\n'.format(cmd)) + time.sleep(time_out) + self._ssh_fd.close() + result = ssh.recv(65535) + except SSHException as e: + raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.host_ip, cmd, e)) + return result.decode('utf-8') + + def get_name(self): + return "remote_{0}".format(self.host_ip) diff --git a/common/ssh_client/ssh.py b/common/ssh_client/ssh.py new file mode 100644 index 00000000..a78e9e6f --- /dev/null +++ b/common/ssh_client/ssh.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@time: 2024/6/24 +@file: ssh.py +@desc: +""" +import socket +from common.ssh_client.docker_client import DockerClient +from common.ssh_client.kubernetes_client import KubernetesClient +from common.ssh_client.local_client import LocalClient +from common.ssh_client.remote_client import RemoteClient +from stdio import SafeStdio + + +class SshClient(SafeStdio): + def __init__(self, context=None, node=None): + if node is None: + raise Exception("SshHelper init error: node is None") + self.node = node + self.context = context + self.ssh_type = node.get("ssh_type") or "remote" + self.client = None + self.init() + + def local_ip(self): + local_ip_list = [] + hostname = socket.gethostname() + addresses = socket.getaddrinfo(hostname, None) + for address in addresses: + local_ip_list.append(address[4][0]) + local_ip_list.append('127.0.0.1') + return list(set(local_ip_list)) + + def init(self): + try: + self.ssh_type = self.node.get("ssh_type") or "remote" + # where ssh_type is remote, maybe use local client. + if self.ssh_type == 'remote' or self.ssh_type == 'ssh': + node_ip = self.node.get("ip") or "" + if node_ip == "": + raise Exception("the node ip is None") + if node_ip in self.local_ip(): + self.ssh_type = "local" + if self.ssh_type == 'local': + self.client = LocalClient(self.context, self.node) + elif self.ssh_type == "remote": + self.client = RemoteClient(self.context, self.node) + elif self.ssh_type == 'docker': + self.client = DockerClient(self.context, self.node) + elif self.ssh_type == 'kubernetes': + self.client = KubernetesClient(self.context, self.node) + else: + raise Exception("the ssh type is not support: {0}".format(self.ssh_type)) + except Exception as e: + raise Exception("init ssh client error: {}".format(e)) + + def exec_cmd(self, cmd): + return self.client.exec_cmd(cmd).strip() + + def download(self, remote_path, local_path): + return self.client.download(remote_path, local_path) + + def upload(self, remote_path, local_path): + return self.client.upload(remote_path, local_path) + + def ssh_invoke_shell_switch_user(self, new_user, cmd, time_out): + return self.client.ssh_invoke_shell_switch_user(new_user, cmd, time_out) + + def ssh_close(self): + return self.client.ssh_close() + + def ssh_reconnect(self): + self.client = None + self.init() + return + + def run(self, cmd): + return self.exec_cmd(cmd) + + def get_name(self): + return self.client.get_name() + + def get_ip(self): + return self.client.get_ip() diff --git a/common/tool.py b/common/tool.py index 0fbe0220..51f9dd9b 100644 --- a/common/tool.py +++ b/common/tool.py @@ -504,11 +504,11 @@ def size_format(num, unit="B", output_str=False, stdio=None): return new_num, units[unit_idx] @staticmethod - def show_file_size_tabulate(ip, file_size, stdio=None): + def show_file_size_tabulate(ssh_client, file_size, stdio=None): format_file_size = FileUtil.size_format(int(file_size), output_str=True, stdio=stdio) summary_tab = [] field_names = ["Node", "LogSize"] - summary_tab.append((ip, format_file_size)) + summary_tab.append((ssh_client.get_name(), format_file_size)) return "\nZipFileInfo:\n" + tabulate.tabulate(summary_tab, headers=field_names, tablefmt="grid", showindex=False) @staticmethod diff --git a/conf/inner_config.yml b/conf/inner_config.yml index 32db3c84..b0005e03 100644 --- a/conf/inner_config.yml +++ b/conf/inner_config.yml @@ -25,9 +25,6 @@ rca: result_path: "./rca/" analyze_sql: output_type: "html" - result_path: "./obdiag_analyze_sql/" - sql_audit_limit: 10 + result_path: "./obdiag_analyze/" + sql_audit_limit: 2000 elapsed_time: 100 -analyze_sql_review: - output_type: "html" - result_path: "./obdiag_analyze_sql_review/" diff --git a/config.py b/config.py index 449d8a22..a9c36bef 100644 --- a/config.py +++ b/config.py @@ -93,13 +93,9 @@ }, 'analyze_sql': { 'output_type': 'html', - 'result_path': './obdiag_analyze_sql/', + 'result_path': './obdiag_analyze/', 'sql_audit_limit': 2000, }, - 'analyze_sql_review': { - 'output_type': 'html', - 'result_path': './obdiag_analyze_sql_review/', - }, } @@ -199,7 +195,10 @@ def create_ob_cluster_node(node_config, global_config): 'redo_dir': node_config.get('redo_dir', global_config.get('redo_dir', '/root/observer/store')), 'ssh_key_file': node_config.get('ssh_key_file', global_config.get('ssh_key_file', '')), 'ssh_type': node_config.get('ssh_type', global_config.get('ssh_type', 'remote')), - 'container_name': node_config.get('container_name', global_config.get('container_name')), + 'container_name': node_config.get('container_name', global_config.get('container_name', '')), + 'namespace': node_config.get('namespace', global_config.get('namespace', '')), + 'pod_name': node_config.get('pod_name', global_config.get('pod_name', '')), + "kubernetes_config_file": node_config.get('kubernetes_config_file', global_config.get('kubernetes_config_file', '')), 'host_type': 'OBSERVER', } @@ -230,8 +229,11 @@ def create_ob_proxy_node(node_config, global_config): 'ssh_port': node_config.get('ssh_port', global_config.get('ssh_port', 22)), 'home_path': node_config.get('home_path', global_config.get('home_path', '/root/obproxy')), 'ssh_key_file': node_config.get('ssh_key_file', global_config.get('ssh_key_file', '')), - 'ssh_type': node_config.get('ssh_type', global_config.get('ssh_type', 'ssh')), + 'ssh_type': node_config.get('ssh_type', global_config.get('ssh_type', 'remote')), 'container_name': node_config.get('container_name', global_config.get('container_name')), + 'namespace': node_config.get('namespace', global_config.get('namespace', '')), + 'pod_name': node_config.get('pod_name', global_config.get('pod_name', '')), + "kubernetes_config_file": node_config.get('kubernetes_config_file', global_config.get('kubernetes_config_file', '')), 'host_type': 'OBPROXY', } diff --git a/core.py b/core.py index 8014c864..9e174b81 100644 --- a/core.py +++ b/core.py @@ -22,10 +22,11 @@ from optparse import Values from copy import copy +from common.ssh_client.remote_client import dis_rsa_algorithms from handler.gather.gather_ash_report import GatherAshReportHandler from handler.rca.rca_handler import RCAHandler from handler.rca.rca_list import RcaScenesListHandler -from common.ssh import SshClient, SshConfig, dis_rsa_algorithms +from common.ssh import SshClient, SshConfig from context import HandlerContextNamespace, HandlerContext from config import ConfigManager, InnerConfigManager from err import CheckStatus, SUG_SSH_FAILED diff --git a/dev_init.sh b/dev_init.sh old mode 100644 new mode 100755 index b9cb8692..d35d83d2 --- a/dev_init.sh +++ b/dev_init.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash PROJECT_PATH=$(cd "$(dirname "$0")"; pwd) WORK_DIR=$(readlink -f "$(dirname ${BASH_SOURCE[0]})") @@ -42,6 +42,8 @@ copy_file(){ mkdir -p ${OBDIAG_HOME}/gather if [ -d "${WORK_DIR}/handler/checker/tasks" ]; then cp -rf ${WORK_DIR}/handler/checker/tasks ${OBDIAG_HOME}/check/ + cp -rf ${WORK_DIR}/handler/checker/tasks/obproxy_check_package.yaml ${OBDIAG_HOME}/check/ + cp -rf ${WORK_DIR}/handler/checker/tasks/observer_check_package.yaml ${OBDIAG_HOME}/check/ fi if [ -d "${WORK_DIR}/handler/gather/tasks" ]; then cp -rf ${WORK_DIR}/handler/gather/tasks ${OBDIAG_HOME}/gather/ @@ -51,8 +53,8 @@ copy_file(){ cp -rf ${WORK_DIR}/example ${OBDIAG_HOME}/ fi - if [ -d "${WORK_DIR}/rca" ]; then - cp -rf ${WORK_DIR}/rca ${OBDIAG_HOME}/ + if [ -d "${WORK_DIR}/handler/rca/scene" ]; then + cp -rf ${WORK_DIR}/handler/rca/scene ${OBDIAG_HOME}/rca fi } diff --git a/diag_cmd.py b/diag_cmd.py new file mode 100644 index 00000000..15d1a298 --- /dev/null +++ b/diag_cmd.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -* +# Copyright (c) 2022 OceanBase +# OceanBase Diagnostic Tool is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +""" +@file: diag_cmd.py +@desc: +""" + +from __future__ import absolute_import, division, print_function +from common.tool import Util + +import os +import sys +import textwrap +import re +from uuid import uuid1 as uuid, UUID +from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter +from core import ObdiagHome +from stdio import IO +from common.version import get_obdiag_version +from telemetry.telemetry import telemetry + +ROOT_IO = IO(1) +OBDIAG_HOME_PATH = os.path.join(os.getenv('HOME'), 'oceanbase-diagnostic-tool') + + +class OptionHelpFormatter(IndentedHelpFormatter): + + def format_option(self, option): + result = [] + opts = self.option_strings[option] + opt_width = self.help_position - self.current_indent - 2 + if len(opts) > opt_width: + opts = "%*s%s\n" % (self.current_indent, "", opts) + indent_first = self.help_position + else: + opts = "%*s%-*s " % (self.current_indent, "", opt_width, opts) + indent_first = 0 + result.append(opts) + if option.help: + help_text = self.expand_default(option) + help_lines = help_text.split('\n') + if len(help_lines) == 1: + help_lines = textwrap.wrap(help_text, self.help_width) + result.append("%*s%s\n" % (indent_first, "", help_lines[0])) + result.extend(["%*s%s\n" % (self.help_position, "", line) for line in help_lines[1:]]) + elif opts[-1] != "\n": + result.append("\n") + return "".join(result) + + +class AllowUndefinedOptionParser(OptionParser): + IS_TTY = sys.stdin.isatty() + + def __init__(self, usage=None, option_list=None, option_class=Option, version=None, conflict_handler="resolve", description=None, formatter=None, add_help_option=True, prog=None, epilog=None, allow_undefine=True, undefine_warn=True): + OptionParser.__init__(self, usage, option_list, option_class, version, conflict_handler, description, formatter, add_help_option, prog, epilog) + self.allow_undefine = allow_undefine + self.undefine_warn = undefine_warn + + def warn(self, msg, file=None): + if self.IS_TTY: + print("%s %s" % (IO.WARNING_PREV, msg)) + else: + print('warn: %s' % msg) + + def _process_long_opt(self, rargs, values): + try: + value = rargs[0] + OptionParser._process_long_opt(self, rargs, values) + except BadOptionError as e: + if self.allow_undefine: + key = e.opt_str + value = value[len(key) + 1 :] + setattr(values, key.strip('-').replace('-', '_'), value if value != '' else True) + self.undefine_warn and self.warn(e) + else: + raise e + + def _process_short_opts(self, rargs, values): + try: + value = rargs[0] + OptionParser._process_short_opts(self, rargs, values) + except BadOptionError as e: + if self.allow_undefine: + key = e.opt_str + value = value[len(key) + 1 :] + setattr(values, key.strip('-').replace('-', '_'), value if value != '' else True) + self.undefine_warn and self.warn(e) + else: + raise e + + +class BaseCommand(object): + + def __init__(self, name, summary): + self.name = name + self.summary = summary + self.args = [] + self.cmds = [] + self.opts = {} + self.prev_cmd = '' + self.is_init = False + self.hidden = False + self.has_trace = True + self.parser = AllowUndefinedOptionParser(add_help_option=True) + self.parser.add_option('-h', '--help', action='callback', callback=self._show_help, help='Show help and exit.') + self.parser.add_option('-v', '--verbose', action='callback', callback=self._set_verbose, help='Activate verbose output.') + + def _set_verbose(self, *args, **kwargs): + ROOT_IO.set_verbose_level(0xFFFFFFF) + + def init(self, cmd, args): + if self.is_init is False: + self.prev_cmd = cmd + self.args = args + self.is_init = True + self.parser.prog = self.prev_cmd + option_list = self.parser.option_list[2:] + option_list.append(self.parser.option_list[0]) + option_list.append(self.parser.option_list[1]) + self.parser.option_list = option_list + return self + + def parse_command(self): + self.opts, self.cmds = self.parser.parse_args(self.args) + return self.opts + + def do_command(self): + raise NotImplementedError + + def _show_help(self, *args, **kwargs): + ROOT_IO.print(self._mk_usage()) + self.parser.exit(0) + + def _mk_usage(self): + return self.parser.format_help(OptionHelpFormatter()) + + +class ObdiagOriginCommand(BaseCommand): + OBDIAG_PATH = OBDIAG_HOME_PATH + + @property + def enable_log(self): + return True + + def is_valid_time_format(self, time_string): + time_pattern = r'^\d{2}:\d{2}:\d{2}$' + return bool(re.match(time_pattern, time_string)) + + def preprocess_argv(self, argv): + """ + Preprocesses the command line arguments to ensure that date-time strings for --from and --to + options are properly quoted, even if they are originally provided without quotes. + """ + processed_argv = [] + from_index = None + to_index = None + for i, arg in enumerate(argv): + if arg == '--from': + from_index = i + 1 + elif arg == '--to': + to_index = i + 1 + + if from_index is not None and i == from_index: + next_arg = argv[i + 1] if i + 1 < len(argv) else None + if next_arg and self.is_valid_time_format(next_arg): + processed_argv.append(argv[i] + ' ' + next_arg) + from_index = None + i += 1 + else: + processed_argv.append(arg) + elif to_index is not None and i == to_index: + next_arg = argv[i + 1] if i + 1 < len(argv) else None + if next_arg and self.is_valid_time_format(next_arg): + processed_argv.append(argv[i] + ' ' + next_arg) + to_index = None + i += 1 + else: + processed_argv.append(arg) + else: + processed_argv.append(arg) + return processed_argv + + def parse_command(self): + self.args = self.preprocess_argv(self.args) + return super(ObdiagOriginCommand, self).parse_command() + + def do_command(self): + self.parse_command() + trace_id = uuid() + ret = False + try: + log_directory = os.path.join(os.path.expanduser("~"), ".obdiag", "log") + if not os.path.exists(log_directory): + os.makedirs(log_directory, exist_ok=True) + log_path = os.path.join(log_directory, 'obdiag.log') + if self.enable_log: + ROOT_IO.init_trace_logger(log_path, 'obdiag', trace_id) + ROOT_IO.track_limit += 1 + ROOT_IO.verbose('cmd: %s' % self.cmds) + ROOT_IO.verbose('opts: %s' % self.opts) + config_path = os.path.expanduser('~/.obdiag/config.yml') + custom_config = Util.get_option(self.opts, 'c') + if custom_config: + if os.path.exists(os.path.abspath(custom_config)): + config_path = custom_config + else: + ROOT_IO.error('The option you provided with -c: {0} is a non-existent configuration file path.'.format(custom_config)) + return + obdiag = ObdiagHome(stdio=ROOT_IO, config_path=config_path) + obdiag.set_options(self.opts) + obdiag.set_cmds(self.cmds) + ret = self._do_command(obdiag) + telemetry.put_data() + except NotImplementedError: + ROOT_IO.exception('command \'%s\' is not implemented' % self.prev_cmd) + except SystemExit: + pass + except KeyboardInterrupt: + ROOT_IO.exception('Keyboard Interrupt') + except: + e = sys.exc_info()[1] + ROOT_IO.exception('Running Error: %s' % e) + if self.has_trace: + ROOT_IO.print('Trace ID: %s' % trace_id) + ROOT_IO.print('If you want to view detailed obdiag logs, please run: {0} display-trace {1}'.format(obdiag_bin, trace_id)) + return ret + + def _do_command(self, obdiag): + raise NotImplementedError + + def get_white_ip_list(self): + if self.opts.white: + return self.opts.white.split(',') + ROOT_IO.warn("Security Risk: the whitelist is empty and anyone can request this program!") + if ROOT_IO.confirm("Do you want to continue?"): + return [] + wthite_ip_list = ROOT_IO.read("Please enter the whitelist, eq: '192.168.1.1'") + raise wthite_ip_list.split(',') + + +class DisplayTraceCommand(ObdiagOriginCommand): + + def __init__(self): + super(DisplayTraceCommand, self).__init__('display-trace', 'display trace_id log.') + self.has_trace = False + + @property + def enable_log(self): + return False + + def _do_command(self, obdiag): + from common.ssh import LocalClient + + if not self.cmds: + return self._show_help() + log_dir = os.path.expanduser('~/.obdiag/log') + trace_id = self.cmds[0] + ROOT_IO.verbose('Get log by trace_id') + try: + if UUID(trace_id).version != 1: + ROOT_IO.critical('%s is not trace id' % trace_id) + return False + except: + ROOT_IO.print('%s is not trace id' % trace_id) + return False + cmd = 'cd {} && grep -h "\[{}\]" $(ls -tr {}*) | sed "s/\[{}\] //g" '.format(log_dir, trace_id, log_dir, trace_id) + data = LocalClient.execute_command(cmd) + ROOT_IO.print(data.stdout) + return True + + +obdiag_bin = "obdiag" + + +class MajorCommand(BaseCommand): + + def __init__(self, name, summary): + super(MajorCommand, self).__init__(name, summary) + self.commands = {} + + def _mk_usage(self): + if self.commands: + usage = ['%s [options]\n\nAvailable commands:\n' % self.prev_cmd] + commands = [x for x in self.commands.values() if not (hasattr(x, 'hidden') and x.hidden)] + commands.sort(key=lambda x: x.name) + for command in commands: + if command.hidden is False: + usage.append("%-12s %s\n" % (command.name, command.summary)) + self.parser.set_usage('\n'.join(usage)) + return super(MajorCommand, self)._mk_usage() + + def do_command(self): + if not self.is_init: + ROOT_IO.error('%s command not init' % self.prev_cmd) + raise SystemExit('command not init') + if len(self.args) < 1: + ROOT_IO.print('You need to give some commands.\n\nTry `obdiag --help` for more information.') + self._show_help() + return False + base, args = self.args[0], self.args[1:] + if base not in self.commands: + self.parse_command() + self._show_help() + return False + cmd = '%s %s' % (self.prev_cmd, base) + ROOT_IO.track_limit += 1 + global obdiag_bin + obdiag_bin_list = cmd.split() + obdiag_bin = obdiag_bin_list[0] + if "main.py" in cmd: + telemetry.work_tag = False + telemetry.push_cmd_info("cmd: {0}. args:{1}".format(cmd, args)) + return self.commands[base].init(cmd, args).do_command() + + def register_command(self, command): + self.commands[command.name] = command + + +class ObdiagGatherAllCommand(ObdiagOriginCommand): + + def init(self, cmd, args): + super(ObdiagGatherAllCommand, self).init(cmd, args) + return self + + def __init__(self): + super(ObdiagGatherAllCommand, self).__init__('all', 'Gather oceanbase diagnostic info') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') + self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherAllCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_all', self.opts) + + +class ObdiagGatherLogCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherLogCommand, self).__init__('log', 'Gather oceanbase logs from oceanbase machines') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') + self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherLogCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_log', self.opts) + + +class ObdiagGatherSysStatCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherSysStatCommand, self).__init__('sysstat', 'Gather Host information') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherSysStatCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_sysstat', self.opts) + + +class ObdiagGatherStackCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherStackCommand, self).__init__('stack', 'Gather stack') + + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherStackCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_obstack', self.opts) + + +class ObdiagGatherPerfCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherPerfCommand, self).__init__('perf', 'Gather perf') + + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('--scope', type='string', help="perf type constrains, choices=[sample, flame, pstack, all]", default='all') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherPerfCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_perf', self.opts) + + +class ObdiagGatherSlogCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherSlogCommand, self).__init__('slog', 'Gather slog') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherSlogCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_slog', self.opts) + + +class ObdiagGatherClogCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherClogCommand, self).__init__('clog', 'Gather clog') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherClogCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_clog', self.opts) + + +class ObdiagGatherAwrCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherAwrCommand, self).__init__('awr', 'Gather ParalleSQL information') + self.parser.add_option('--cluster_name', type='string', help='cluster_name from ocp') + self.parser.add_option('--cluster_id', type='string', help='cluster_id from ocp') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherAwrCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_awr', self.opts) + + +class ObdiagGatherPlanMonitorCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherPlanMonitorCommand, self).__init__('plan_monitor', 'Gather ParalleSQL information') + self.parser.add_option('--trace_id', type='string', help='sql trace id') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('--env', type='string', help='''env, eg: "{db_connect='-h127.0.0.1 -P2881 -utest@test -p****** -Dtest'}"''') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherPlanMonitorCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_plan_monitor', self.opts) + + +class ObdiagGatherObproxyLogCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherObproxyLogCommand, self).__init__('obproxy_log', 'Gather obproxy log from obproxy machines') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[obproxy, obproxy_limit, obproxy_stat, obproxy_digest, obproxy_slow, obproxy_diagnosis, obproxy_error, all]", default='all') + self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherObproxyLogCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.gather_obproxy_log(self.opts) + + +class ObdiagGatherSceneListCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherSceneListCommand, self).__init__('list', 'gather scene list') + + def init(self, cmd, args): + super(ObdiagGatherSceneListCommand, self).init(cmd, args) + return self + + def _do_command(self, obdiag): + return obdiag.gather_scenes_list(self.opts) + + +class ObdiagGatherSceneRunCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherSceneRunCommand, self).__init__('run', 'gather scene run') + self.parser.add_option('--scene', type='string', help="Specify the scene to be gather") + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--env', type='string', help='env, eg: "{env1=xxx, env2=xxx}"') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherSceneRunCommand, self).init(cmd, args) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_scenes_run', self.opts) + + +class ObdiagGatherAshReportCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagGatherAshReportCommand, self).__init__('ash', 'Gather ash report') + self.parser.add_option('--trace_id', type='string', help="The TRACE.ID of the SQL to be sampled, if left blank or filled with NULL, indicates that TRACE.ID is not restricted.") + self.parser.add_option('--sql_id', type='string', help="The SQL.ID, if left blank or filled with NULL, indicates that SQL.ID is not restricted.") + # WAIT_CLASS + self.parser.add_option('--wait_class', type='string', help='Event types to be sampled.') + self.parser.add_option('--report_type', type='string', help='Report type, currently only supports text type.', default='TEXT') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagGatherAshReportCommand, self).init(cmd, args) + return self + + def _do_command(self, obdiag): + return obdiag.gather_function('gather_ash_report', self.opts) + + +class ObdiagAnalyzeLogCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagAnalyzeLogCommand, self).__init__('log', 'Analyze oceanbase log from online observer machines or offline oceanbase log files') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]", default='all') + self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain") + self.parser.add_option('--log_level', type='string', help="oceanbase logs greater than or equal to this level will be analyze, choices=[DEBUG, TRACE, INFO, WDIAG, WARN, EDIAG, ERROR]") + self.parser.add_option('--files', action="append", type='string', help="specify files") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + + def init(self, cmd, args): + super(ObdiagAnalyzeLogCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + offline_args_sign = '--files' + if self.args and (offline_args_sign in self.args): + return obdiag.analyze_fuction('analyze_log_offline', self.opts) + else: + return obdiag.analyze_fuction('analyze_log', self.opts) + + +class ObdiagAnalyzeFltTraceCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagAnalyzeFltTraceCommand, self).__init__('flt_trace', 'Analyze oceanbase trace.log from online observer machines or offline oceanbase trace.log files') + self.parser.add_option('--flt_trace_id', type='string', help="flt trace id, . format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx") + self.parser.add_option('--files', action="append", help="specify files") + self.parser.add_option('--top', type='string', help="top leaf span", default=5) + self.parser.add_option('--recursion', type='string', help="Maximum number of recursion", default=8) + self.parser.add_option('--output', type='string', help="Print the result to the maximum output line on the screen", default=60) + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeFltTraceCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_flt_trace', self.opts) + + +class ObdiagAnalyzeSQLCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagAnalyzeSQLCommand, self).__init__('sql', 'Analyze oceanbase sql from sql_audit ') + self.parser.add_option('--host', type='string', help="tenant connection host") + self.parser.add_option('--port', type='string', help="tenant connection port") + self.parser.add_option('--password', type='string', help="tenant connection user password", default='') + self.parser.add_option('--user', type='string', help="tenant connection user name") + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.", default='30m') + self.parser.add_option('--level', type='string', help="The alarm level, optional parameters [critical, warn, notice, ok]", default='notice') + self.parser.add_option('--output', type='string', help="The format of the output results, choices=[json, html]", default='html') + self.parser.add_option('--limit', type='string', help="The limit on the number of data rows returned by sql_audit for the tenant.", default=20000) + self.parser.add_option('--store_dir', type='string', help='the dir to store result, current dir by default.', default='./obdiag_analyze/') + self.parser.add_option('--elapsed_time', type='string', help='The minimum threshold for filtering execution time, measured in microseconds.', default=100000) + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeSQLCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_sql', self.opts) + + +class ObdiagAnalyzeSQLReviewCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagAnalyzeSQLReviewCommand, self).__init__('sql_review', 'Analyze oceanbase sql from sql_audit ') + self.parser.add_option('--host', type='string', help="tenant connection host") + self.parser.add_option('--port', type='string', help="tenant connection port") + self.parser.add_option('--password', type='string', help="tenant connection user password", default='') + self.parser.add_option('--user', type='string', help="tenant connection user name") + self.parser.add_option('--files', type='string', action="append", help="specify files") + self.parser.add_option('--level', type='string', help="The alarm level, optional parameters [critical, warn, notice, ok]", default='notice') + self.parser.add_option('--output', type='string', help="The format of the output results, choices=[json, html]", default='html') + self.parser.add_option('--store_dir', type='string', help='the dir to store result, current dir by default.', default='./obdiag_analyze/') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagAnalyzeSQLReviewCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.analyze_fuction('analyze_sql_review', self.opts) + + +class ObdiagCheckCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagCheckCommand, self).__init__('check', 'check oceanbase cluster') + self.parser.add_option('--cases', type='string', help="check observer's cases on package_file") + self.parser.add_option('--obproxy_cases', type='string', help="check obproxy's cases on package_file") + self.parser.add_option('--store_dir', type='string', help='the dir to store check result, current dir by default.', default='./check_report/') + self.parser.add_option('--report_type', type='string', help='The type of the check report, support "table", "json", "xml", "yaml". default table', default='table') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagCheckCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + if 'list' in self.args: + obdiag.check_list(self.opts) + return + return obdiag.check(self.opts) + + +class ObdiagRCARunCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagRCARunCommand, self).__init__('run', 'root cause analysis') + self.parser.add_option('--scene', type='string', help="rca scene name. The argument is required.") + self.parser.add_option('--store_dir', type='string', help='the dir to store rca result, current dir by default.', default='./rca/') + self.parser.add_option('--input_parameters', type='string', help='input parameters of scene') + self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml')) + + def init(self, cmd, args): + super(ObdiagRCARunCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.rca_run(self.opts) + + +class ObdiagRCAListCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagRCAListCommand, self).__init__('list', 'show list of rca list') + + def init(self, cmd, args): + super(ObdiagRCAListCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.rca_list(self.opts) + + +class ObdiagConfigCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagConfigCommand, self).__init__('config', 'Quick build config') + self.parser.add_option('-h', type='string', help="database host") + self.parser.add_option('-u', type='string', help='sys_user', default='root@sys') + self.parser.add_option('-p', type='string', help="password", default='') + self.parser.add_option('-P', type='string', help="port") + + def init(self, cmd, args): + super(ObdiagConfigCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.config(self.opts) + + +class ObdiagUpdateCommand(ObdiagOriginCommand): + + def __init__(self): + super(ObdiagUpdateCommand, self).__init__('update', 'Update cheat files') + self.parser.add_option('--file', type='string', help="obdiag update cheat file path. Please note that you need to ensure the reliability of the files on your own.") + self.parser.add_option( + '--force', + action='store_true', + help='You can force online upgrades by adding --force in the command', + ) + + def init(self, cmd, args): + super(ObdiagUpdateCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obdiag): + return obdiag.update(self.opts) + + +class ObdiagGatherCommand(MajorCommand): + + def __init__(self): + super(ObdiagGatherCommand, self).__init__('gather', 'Gather oceanbase diagnostic info') + self.register_command(ObdiagGatherAllCommand()) + self.register_command(ObdiagGatherLogCommand()) + self.register_command(ObdiagGatherSysStatCommand()) + self.register_command(ObdiagGatherStackCommand()) + self.register_command(ObdiagGatherPerfCommand()) + self.register_command(ObdiagGatherSlogCommand()) + self.register_command(ObdiagGatherClogCommand()) + self.register_command(ObdiagGatherPlanMonitorCommand()) + self.register_command(ObdiagGatherAwrCommand()) + self.register_command(ObdiagGatherObproxyLogCommand()) + self.register_command(ObdiagGatherSceneCommand()) + self.register_command(ObdiagGatherAshReportCommand()) + + +class ObdiagGatherSceneCommand(MajorCommand): + + def __init__(self): + super(ObdiagGatherSceneCommand, self).__init__('scene', 'Gather scene diagnostic info') + self.register_command(ObdiagGatherSceneListCommand()) + self.register_command(ObdiagGatherSceneRunCommand()) + + +class ObdiagAnalyzeCommand(MajorCommand): + + def __init__(self): + super(ObdiagAnalyzeCommand, self).__init__('analyze', 'Analyze oceanbase diagnostic info') + self.register_command(ObdiagAnalyzeLogCommand()) + self.register_command(ObdiagAnalyzeFltTraceCommand()) + self.register_command(ObdiagAnalyzeSQLCommand()) + self.register_command(ObdiagAnalyzeSQLReviewCommand()) + + +class ObdiagRCACommand(MajorCommand): + + def __init__(self): + super(ObdiagRCACommand, self).__init__('rca', 'root cause analysis') + self.register_command(ObdiagRCARunCommand()) + self.register_command(ObdiagRCAListCommand()) + + +class MainCommand(MajorCommand): + + def __init__(self): + super(MainCommand, self).__init__('obdiag', '') + self.register_command(DisplayTraceCommand()) + self.register_command(ObdiagGatherCommand()) + self.register_command(ObdiagAnalyzeCommand()) + self.register_command(ObdiagCheckCommand()) + self.register_command(ObdiagRCACommand()) + self.register_command(ObdiagConfigCommand()) + self.register_command(ObdiagUpdateCommand()) + self.parser.version = get_obdiag_version() + self.parser._add_version_option() diff --git a/example/operator.yml b/example/operator.yml new file mode 100644 index 00000000..05c997ee --- /dev/null +++ b/example/operator.yml @@ -0,0 +1,37 @@ +obcluster: + db_host: 127.0.0.1 + db_port: 2881 # 2881 + ob_cluster_name: obtest + tenant_sys: + user: root@sys + password: '' + servers: + nodes: + - namespace: oceanbase + ssh_type: "kubernetes" + pod_name: obcluster-1-zone1-xxxx + container_name: observer + home_path: /home/admin/oceanbase + data_dir: /home/admin/oceanbase/store + redo_dir: /home/admin/oceanbase/store + ip: xx.xx.xx.xx + - namespace: oceanbase + ssh_type: "kubernetes" + pod_name: obcluster-1-zone2-xxxx + container_name: observer + home_path: /home/admin/oceanbase + data_dir: /home/admin/oceanbase/store + redo_dir: /home/admin/oceanbase/store + ip: xx.xx.xx.xx + - namespace: oceanbase + ssh_type: "kubernetes" + pod_name: obcluster-1-zone3-xxxx + container_name: observer + home_path: /home/admin/oceanbase + data_dir: /home/admin/oceanbase/store + redo_Dir: /home/admin/oceanbase/store + ip: xx.xx.xx.xx + global: + # if running obdiag in kubernetes, please delete the kubernetes_config_file + kubernetes_config_file: "~/.kube/config/config.yaml" + diff --git a/handler/analyzer/analyze_flt_trace.py b/handler/analyzer/analyze_flt_trace.py index 1c338818..fb14ccae 100644 --- a/handler/analyzer/analyze_flt_trace.py +++ b/handler/analyzer/analyze_flt_trace.py @@ -24,7 +24,6 @@ from common.command import LocalClient, SshClient, delete_file from handler.analyzer.log_parser.tree import Tree from common.command import download_file, mkdir -from common.ssh import SshHelper from common.tool import TimeUtils from common.tool import Util from common.tool import DirectoryUtil @@ -139,30 +138,31 @@ def __handle_from_node(self, node, old_files, local_store_parent_dir): local_store_dir = "{0}/{1}".format(local_store_parent_dir, remote_ip) DirectoryUtil.mkdir(path=local_store_dir, stdio=self.stdio) ssh_failed = False + ssh_client = None try: - ssh = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node) + ssh_client = SshClient(self.context, node) except Exception as e: ssh = None self.stdio.exception("ssh {0}@{1}: failed, Please check the {2}".format(remote_user, remote_ip, self.config_path)) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) + return resp, node_files if not ssh_failed: gather_dir_name = "trace_merged_cache" gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name) - mkdir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) + mkdir(ssh_client, gather_dir_full_path, self.stdio) if self.is_ssh: - self.__get_online_log_file(ssh, node, gather_dir_full_path, local_store_dir) + self.__get_online_log_file(ssh_client, node, gather_dir_full_path, local_store_dir) else: - self.__get_offline_log_file(ssh, gather_dir_full_path, local_store_dir) - delete_file(self.is_ssh, ssh, os.path.join(gather_dir_full_path, str(node.get("host_type")) + '-' + str(self.flt_trace_id)), self.stdio) - ssh.ssh_close() + self.__get_offline_log_file(ssh_client, gather_dir_full_path, local_store_dir) + delete_file(ssh_client, os.path.join(gather_dir_full_path, str(node.get("host_type")) + '-' + str(self.flt_trace_id)), self.stdio) for file in FileUtil.find_all_file(local_store_dir): if self.flt_trace_id in file and (file not in old_files): node_files.append(file) return resp, node_files - def __get_online_log_file(self, ssh_helper, node, gather_path, local_store_dir): + def __get_online_log_file(self, ssh_client, node, gather_path, local_store_dir): """ :param ssh_helper, log_name, gather_path :return: @@ -188,13 +188,13 @@ def check_filename(filename): local_store_path = check_filename(local_store_path) grep_cmd = "grep '{grep_args}' {log_dir}/*trace.log* > {gather_path}/{log_name} ".format(grep_args=self.flt_trace_id, gather_path=gather_path, log_name=self.flt_trace_id, log_dir=log_path) self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) + ssh_client.exec_cmd(grep_cmd) log_full_path = "{gather_path}/{log_name}".format(log_name=self.flt_trace_id, gather_path=gather_path) - download_file(True, ssh_helper, log_full_path, local_store_path, self.stdio) + download_file(ssh_client, log_full_path, local_store_path, self.stdio) - def __get_offline_log_file(self, ssh_helper, log_full_path, local_store_dir): + def __get_offline_log_file(self, ssh_client, log_full_path, local_store_dir): """ - :param ssh_helper, log_name + :param ssh_client, log_name :return: """ local_store_path = os.path.join(local_store_dir, self.flt_trace_id) @@ -202,7 +202,7 @@ def __get_offline_log_file(self, ssh_helper, log_full_path, local_store_dir): if self.flt_trace_id is not None and (len(log_name_list) > 0): grep_cmd = "grep -e '{grep_args}' {log_file} > {local_store_path} ".format(grep_args=self.flt_trace_id, log_file=' '.join(log_name_list), local_store_path=local_store_path) LocalClient(self.stdio).run(grep_cmd) - download_file(False, ssh_helper, log_full_path, local_store_path, self.stdio) + download_file(ssh_client, log_full_path, local_store_path, self.stdio) def __get_log_name_list_offline(self): """ diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index 3ec21ade..0d4a9646 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -27,7 +27,6 @@ from common.ob_log_level import OBLogLevel from handler.meta.ob_error import OB_RET_DICT from common.command import download_file, get_logfile_name_list, mkdir, delete_file -from common.ssh import SshHelper from common.tool import Util from common.tool import DirectoryUtil from common.tool import FileUtil @@ -132,20 +131,9 @@ def handle(self): local_store_parent_dir = os.path.join(self.gather_pack_dir, "obdiag_analyze_pack_{0}".format(TimeUtils.timestamp_to_filename_time(TimeUtils.get_current_us_timestamp()))) self.stdio.verbose("Use {0} as pack dir.".format(local_store_parent_dir)) analyze_tuples = [] - - def handle_from_node(node): + for node in self.nodes: resp, node_results = self.__handle_from_node(node, local_store_parent_dir) analyze_tuples.append((node.get("ip"), False, resp["error"], node_results)) - - if self.is_ssh: - for node in self.nodes: - handle_from_node(node) - else: - local_ip = '127.0.0.1' - node = self.nodes[0] - node["ip"] = local_ip - handle_from_node(node) - self.stdio.start_loading('analyze result start') title, field_names, summary_list, summary_details_list = self.__get_overall_summary(analyze_tuples, self.directly_analyze_files) table = tabulate.tabulate(summary_list, headers=field_names, tablefmt="grid", showindex=False) @@ -164,60 +152,55 @@ def handle_from_node(node): def __handle_from_node(self, node, local_store_parent_dir): resp = {"skip": False, "error": ""} - node_results = [] - remote_ip = node.get("ip") if self.is_ssh else '127.0.0.1' - remote_user = node.get("ssh_username") - remote_password = node.get("ssh_password") - remote_port = node.get("ssh_port") - remote_private_key = node.get("ssh_key_file") - remote_home_path = node.get("home_path") - self.stdio.verbose("Sending Collect Shell Command to node {0} ...".format(remote_ip)) - DirectoryUtil.mkdir(path=local_store_parent_dir, stdio=self.stdio) - if "ssh_type" in node and node["ssh_type"] == "docker": - local_store_dir = "{0}/docker_{1}".format(local_store_parent_dir, node["container_name"]) - else: - local_store_dir = "{0}/{1}".format(local_store_parent_dir, remote_ip.replace(".", "_")) - DirectoryUtil.mkdir(path=local_store_dir, stdio=self.stdio) - ssh_failed = False - ssh = None + ssh_client = SshClient(self.context, node) try: - ssh = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + node_results = [] + remote_ip = node.get("ip") if self.is_ssh else '127.0.0.1' + remote_user = node.get("ssh_username") + remote_password = node.get("ssh_password") + remote_port = node.get("ssh_port") + remote_private_key = node.get("ssh_key_file") + remote_home_path = node.get("home_path") + self.stdio.verbose("Sending Collect Shell Command to node {0} ...".format(remote_ip)) + DirectoryUtil.mkdir(path=local_store_parent_dir, stdio=self.stdio) + local_store_dir = "{0}/{1}".format(local_store_parent_dir, ssh_client.get_name()) + DirectoryUtil.mkdir(path=local_store_dir, stdio=self.stdio) except Exception as e: - self.stdio.error("ssh {0}@{1}: failed, Please check the {2}".format(remote_user, remote_ip, self.config_path)) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) - if not ssh_failed: - from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str)) - to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str)) - gather_dir_name = "ob_log_{0}_{1}_{2}".format(ssh.host_ip, from_datetime_timestamp, to_datetime_timestamp) - gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name) - mkdir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) + raise Exception("Please check the {0}".format(self.config_path)) + + from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str)) + to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str)) + gather_dir_name = "ob_log_{0}_{1}_{2}".format(ssh_client.get_name(), from_datetime_timestamp, to_datetime_timestamp) + gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name) + mkdir(ssh_client, gather_dir_full_path) - log_list, resp = self.__handle_log_list(ssh, node, resp) - if resp["skip"]: - return resp, node_results - self.stdio.print(FileUtil.show_file_list_tabulate(remote_ip, log_list, self.stdio)) - for log_name in log_list: - if self.directly_analyze_files: - self.__pharse_offline_log_file(ssh_helper=ssh, log_name=log_name, local_store_dir=local_store_dir) - analyze_log_full_path = "{0}/{1}".format(local_store_dir, str(log_name).strip(".").replace("/", "_")) - else: - self.__pharse_log_file(ssh_helper=ssh, node=node, log_name=log_name, gather_path=gather_dir_full_path, local_store_dir=local_store_dir) - analyze_log_full_path = "{0}/{1}".format(local_store_dir, log_name) - self.stdio.start_loading('analyze log start') - file_result = self.__parse_log_lines(analyze_log_full_path) - self.stdio.stop_loading('analyze log sucess') - node_results.append(file_result) - delete_file(self.is_ssh, ssh, gather_dir_full_path, self.stdio) - ssh.ssh_close() + log_list, resp = self.__handle_log_list(ssh_client, node, resp) + if resp["skip"]: + return resp, node_results + self.stdio.print(FileUtil.show_file_list_tabulate(remote_ip, log_list, self.stdio)) + for log_name in log_list: + if self.directly_analyze_files: + self.__pharse_offline_log_file(ssh_client, log_name=log_name, local_store_dir=local_store_dir) + analyze_log_full_path = "{0}/{1}".format(local_store_dir, str(log_name).strip(".").replace("/", "_")) + else: + self.__pharse_log_file(ssh_client, node=node, log_name=log_name, gather_path=gather_dir_full_path, local_store_dir=local_store_dir) + analyze_log_full_path = "{0}/{1}".format(local_store_dir, log_name) + self.stdio.start_loading('analyze log start') + file_result = self.__parse_log_lines(analyze_log_full_path) + self.stdio.stop_loading('analyze log sucess') + node_results.append(file_result) + delete_file(ssh_client, gather_dir_full_path, self.stdio) + ssh_client.ssh_close() return resp, node_results - def __handle_log_list(self, ssh, node, resp): + def __handle_log_list(self, ssh_client, node, resp): if self.directly_analyze_files: log_list = self.__get_log_name_list_offline() else: - log_list = self.__get_log_name_list(ssh, node) + log_list = self.__get_log_name_list(ssh_client, node) if len(log_list) > self.file_number_limit: self.stdio.warn("{0} The number of log files is {1}, out of range (0,{2}]".format(node.get("ip"), len(log_list), self.file_number_limit)) resp["skip"] = (True,) @@ -232,9 +215,9 @@ def __handle_log_list(self, ssh, node, resp): return log_list, resp return log_list, resp - def __get_log_name_list(self, ssh_helper, node): + def __get_log_name_list(self, ssh_client, node): """ - :param ssh_helper: + :param ssh_client: :return: log_name_list """ home_path = node.get("home_path") @@ -244,9 +227,9 @@ def __get_log_name_list(self, ssh_helper, node): else: get_oblog = "ls -1 -F %s/observer.log* %s/rootservice.log* %s/election.log* | awk -F '/' '{print $NF}'" % (log_path, log_path, log_path) log_name_list = [] - log_files = SshClient(self.stdio).run(ssh_helper, get_oblog) if self.is_ssh else LocalClient(self.stdio).run(get_oblog) + log_files = ssh_client.exec_cmd(get_oblog) if log_files: - log_name_list = get_logfile_name_list(self.is_ssh, ssh_helper, self.from_time_str, self.to_time_str, log_path, log_files, self.stdio) + log_name_list = get_logfile_name_list(ssh_client, self.from_time_str, self.to_time_str, log_path, log_files, self.stdio) else: self.stdio.error("Unable to find the log file. Please provide the correct --ob_install_dir, the default is [/home/admin/oceanbase]") return log_name_list @@ -269,33 +252,29 @@ def __get_log_name_list_offline(self): self.stdio.verbose("get log list {}".format(log_name_list)) return log_name_list - def __pharse_log_file(self, ssh_helper, node, log_name, gather_path, local_store_dir): - """ - :param ssh_helper, log_name, gather_path - :return: - """ + def __pharse_log_file(self, ssh_client, node, log_name, gather_path, local_store_dir): home_path = node.get("home_path") log_path = os.path.join(home_path, "log") local_store_path = "{0}/{1}".format(local_store_dir, log_name) if self.grep_args is not None: grep_cmd = "grep -e '{grep_args}' {log_dir}/{log_name} >> {gather_path}/{log_name} ".format(grep_args=self.grep_args, gather_path=gather_path, log_name=log_name, log_dir=log_path) self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) + ssh_client.exec_cmd(grep_cmd) log_full_path = "{gather_path}/{log_name}".format(log_name=log_name, gather_path=gather_path) - download_file(self.is_ssh, ssh_helper, log_full_path, local_store_path, self.stdio) + download_file(ssh_client, log_full_path, local_store_path, self.stdio) else: real_time_logs = ["observer.log", "rootservice.log", "election.log", "trace.log", "observer.log.wf", "rootservice.log.wf", "election.log.wf", "trace.log.wf"] if log_name in real_time_logs: cp_cmd = "cp {log_dir}/{log_name} {gather_path}/{log_name} ".format(gather_path=gather_path, log_name=log_name, log_dir=log_path) self.stdio.verbose("copy files, run cmd = [{0}]".format(cp_cmd)) - SshClient(self.stdio).run(ssh_helper, cp_cmd) if self.is_ssh else LocalClient(self.stdio).run(cp_cmd) + ssh_client.exec_cmd(cp_cmd) log_full_path = "{gather_path}/{log_name}".format(log_name=log_name, gather_path=gather_path) - download_file(self.is_ssh, ssh_helper, log_full_path, local_store_path, self.stdio) + download_file(ssh_client, log_full_path, local_store_path, self.stdio) else: log_full_path = "{log_dir}/{log_name}".format(log_name=log_name, log_dir=log_path) - download_file(self.is_ssh, ssh_helper, log_full_path, local_store_path, self.stdio) + download_file(ssh_client, log_full_path, local_store_path, self.stdio) - def __pharse_offline_log_file(self, ssh_helper, log_name, local_store_dir): + def __pharse_offline_log_file(self, ssh_client, log_name, local_store_dir): """ :param ssh_helper, log_name :return: @@ -304,9 +283,9 @@ def __pharse_offline_log_file(self, ssh_helper, log_name, local_store_dir): if self.grep_args is not None: grep_cmd = "grep -e '{grep_args}' {log_name} >> {local_store_path} ".format(grep_args=self.grep_args, log_name=log_name, local_store_path=local_store_path) self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) + ssh_client.exec_cmd(ssh_client, grep_cmd) else: - download_file(self.is_ssh, ssh_helper, log_name, local_store_path, self.stdio) + download_file(ssh_client, log_name, local_store_path, self.stdio) def __get_observer_ret_code(self, log_line): """ diff --git a/handler/analyzer/analyze_sql.py b/handler/analyzer/analyze_sql.py index 39fd7bec..ad8a7fac 100644 --- a/handler/analyzer/analyze_sql.py +++ b/handler/analyzer/analyze_sql.py @@ -42,10 +42,12 @@ def __init__(self, context): self.to_timestamp = None self.config_path = const.DEFAULT_CONFIG_PATH self.db_connector_provided = False - self.tenant_name = "all" + self.tenant_name = 'all' self.db_user = None - self.sql_audit_limit = 10 - self.elapsed_time = 100 + self.local_stored_parrent_path = os.path.abspath('./obdiag_analyze/') + self.sql_audit_limit = 2000 + self.elapsed_time = 100000 + self.output_type = 'html' self.level = 'notice' self.ob_version = '4.0.0.0' self.sql_audit_keys = [ @@ -106,16 +108,6 @@ def init_inner_config(self): self.inner_config = self.context.inner_config basic_config = self.inner_config['obdiag']['basic'] self.config_path = basic_config['config_path'] - self.local_stored_parrent_path = self.inner_config['analyze_sql']['result_path'] - sql_audit_limit = int(self.inner_config['analyze_sql']['sql_audit_limit']) - if sql_audit_limit: - self.sql_audit_limit = sql_audit_limit - elapsed_time = int(self.inner_config['analyze_sql']['elapsed_time']) - if elapsed_time: - self.elapsed_time = elapsed_time - if not os.path.exists(os.path.abspath(self.local_stored_parrent_path)): - self.stdio.warn('No such directory {0}, Now create it'.format(os.path.abspath(self.local_stored_parrent_path))) - os.makedirs(os.path.abspath(self.local_stored_parrent_path)) return True def init_config(self): @@ -154,6 +146,21 @@ def init_option(self): level_option = Util.get_option(options, 'level') if level_option: self.level = level_option + store_dir_option = Util.get_option(options, 'store_dir') + if store_dir_option is not None: + if not os.path.exists(os.path.abspath(store_dir_option)): + self.stdio.warn('Error: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + os.makedirs(os.path.abspath(store_dir_option)) + self.local_stored_parrent_path = os.path.abspath(store_dir_option) + output_option = Util.get_option(options, 'output') + if output_option: + self.output_type = output_option + limit_option = Util.get_option(options, 'limit') + if limit_option: + self.sql_audit_limit = limit_option + elapsed_time_option = Util.get_option(options, 'elapsed_time') + if elapsed_time_option: + self.elapsed_time = elapsed_time_option if from_option is not None and to_option is not None: try: from_timestamp = TimeUtils.parse_time_str(from_option) @@ -198,17 +205,18 @@ def handle(self): self.stdio.error('init ob version failed') return False self.__init_db_connector() - self.local_store_dir = os.path.join(self.local_stored_parrent_path, "sql_{0}".format(TimeUtils.timestamp_to_filename_time(TimeUtils.get_current_us_timestamp()))) - if not os.path.exists(os.path.abspath(self.local_store_dir)): - os.makedirs(os.path.abspath(self.local_store_dir)) - self.stdio.print("Use {0} as result dir.".format(self.local_store_dir)) + self.local_store_path = os.path.join(self.local_stored_parrent_path, "obdiag_analyze_sql_result_{0}_{1}.html".format(TimeUtils.timestamp_to_filename_time(self.from_timestamp), TimeUtils.timestamp_to_filename_time(self.to_timestamp))) + self.stdio.print("Use {0} as result store path.".format(self.local_store_path)) raw_results = self.__select_sql_audit() results = self.__filter_max_elapsed_time_with_same_sql_id(raw_results) for item in results: item['planCachePlanExplain'] = self.__get_plan_cache_plan_explain(item) item['diagnosticEntries'] = self.__parse_sql_review(item["querySql"]) - html_result = self.__generate_html_result(results) - FileUtil.write_append(os.path.join(self.local_store_dir, "sql_analyze_result.html"), html_result) + if self.output_type == "html": + html_result = self.__generate_html_result(results) + FileUtil.write_append(self.local_store_path, html_result) + else: + pass self.__print_result() def __extract_tenant_name(self, username): @@ -327,4 +335,4 @@ def __generate_html_result(self, all_results): return full_html def __print_result(self): - self.stdio.print(Fore.YELLOW + "\nAnalyze sql results stored in this directory: {0}\n".format(self.local_store_dir) + Style.RESET_ALL) + self.stdio.print(Fore.YELLOW + "\nAnalyze sql results stored in this directory: {0}\n".format(self.local_store_path) + Style.RESET_ALL) diff --git a/handler/analyzer/analyze_sql_review.py b/handler/analyzer/analyze_sql_review.py index b78eb6c3..0fb653b7 100644 --- a/handler/analyzer/analyze_sql_review.py +++ b/handler/analyzer/analyze_sql_review.py @@ -40,16 +40,14 @@ def __init__(self, context): self.analyze_files_list = None self.directly_analyze_files = False self.level = 'notice' + self.local_store_path = None + self.output_type = 'html' def init_inner_config(self): self.stdio.verbose("init inner config start") self.inner_config = self.context.inner_config basic_config = self.inner_config['obdiag']['basic'] self.config_path = basic_config['config_path'] - self.local_stored_parrent_path = self.inner_config['analyze_sql_review']['result_path'] - if not os.path.exists(os.path.abspath(self.local_stored_parrent_path)): - self.stdio.warn('No such directory {0}, Now create it'.format(os.path.abspath(self.local_stored_parrent_path))) - os.makedirs(os.path.abspath(self.local_stored_parrent_path)) self.stdio.verbose("init inner config success") return True @@ -85,6 +83,15 @@ def init_option(self): level_option = Util.get_option(options, 'level') if level_option: self.level = level_option + store_dir_option = Util.get_option(options, 'store_dir') + if store_dir_option is not None: + if not os.path.exists(os.path.abspath(store_dir_option)): + self.stdio.warn('Error: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option))) + os.makedirs(os.path.abspath(store_dir_option)) + self.local_stored_parrent_path = os.path.abspath(store_dir_option) + output_option = Util.get_option(options, 'output') + if output_option: + self.output_type = output_option self.db_user = db_user_option self.db_password = db_password_option return True @@ -100,17 +107,15 @@ def handle(self): self.stdio.error('init config failed') return False self.__init_db_connector() - self.local_store_dir = os.path.join(self.local_stored_parrent_path, "sql_{0}".format(TimeUtils.timestamp_to_filename_time(TimeUtils.get_current_us_timestamp()))) - if not os.path.exists(os.path.abspath(self.local_store_dir)): - os.makedirs(os.path.abspath(self.local_store_dir)) - self.stdio.print("Use {0} as result dir.".format(self.local_store_dir)) - if self.directly_analyze_files: - all_results = self.__directly_analyze_files() - results = self.__parse_results(all_results) + self.local_store_path = os.path.join(self.local_stored_parrent_path, "obdiag_sql_review_result_{0}.html".format(TimeUtils.timestamp_to_filename_time(TimeUtils.get_current_us_timestamp()))) + self.stdio.print("Use {0} as result store path.".format(self.local_store_path)) + all_results = self.__directly_analyze_files() + results = self.__parse_results(all_results) + if self.output_type == "html": html_result = self.__generate_html_result(results) - FileUtil.write_append(os.path.join(self.local_store_dir, "sql_review_result.html"), html_result) + FileUtil.write_append(self.local_store_path, html_result) else: - all_results = self.__analyze_sql_audit() + pass self.__print_result() def __directly_analyze_files(self): @@ -129,9 +134,6 @@ def __directly_analyze_files(self): file_results[file] = sql_results return file_results - def __analyze_sql_audit(self): - return {} - def __get_sql_file_list(self): """ :param: @@ -219,4 +221,4 @@ def __generate_html_result(self, all_results): return full_html def __print_result(self): - self.stdio.print(Fore.YELLOW + "\nAnalyze sql_review results stored in this directory: {0}\n".format(self.local_store_dir) + Style.RESET_ALL) + self.stdio.print(Fore.YELLOW + "\nAnalyze sql_review results stored in this directory: {0}\n".format(self.local_store_path) + Style.RESET_ALL) diff --git a/handler/checker/check_handler.py b/handler/checker/check_handler.py index 992318f0..e74b9ccf 100644 --- a/handler/checker/check_handler.py +++ b/handler/checker/check_handler.py @@ -23,11 +23,11 @@ import yaml from common.ob_connector import OBConnector -from common.ssh import SshHelper +from common.scene import get_version_by_type +from common.ssh_client.ssh import SshClient from handler.checker.check_exception import CheckException from handler.checker.check_report import TaskReport, CheckReport, CheckrReportException from handler.checker.check_task import TaskBase -from common.scene import get_version import re from common.tool import Util from common.tool import YamlUtils @@ -101,13 +101,14 @@ def __init__(self, context, check_target_type="observer"): # add ssher ssher = None try: - ssher = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) + # ssher = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) + ssher = SshClient(context, node) except Exception as e: self.stdio.warn("StepBase get SshHelper fail on{0} ,Exception: {1}".format(node.get("ip"), e)) node["ssher"] = ssher new_node.append(node) self.nodes = new_node - self.version = get_version(self.nodes, self.check_target_type, self.cluster, self.stdio) + self.version = get_version_by_type(self.context, self.check_target_type, self.stdio) obConnectorPool = None # add OBConnectorPool try: @@ -196,7 +197,7 @@ def get_all_tasks(self): # need package_name def get_package_tasks(self, package_name): # Obtain information within the package file - with open(self.package_file_name, 'r') as file: + with open(self.package_file_name, 'r', encoding='utf-8') as file: package_file_data = yaml.safe_load(file) packege_tasks = package_file_data if package_name not in packege_tasks: diff --git a/handler/checker/result/verify.py b/handler/checker/result/verify.py index e59d7318..06dde6b0 100644 --- a/handler/checker/result/verify.py +++ b/handler/checker/result/verify.py @@ -74,7 +74,7 @@ def _verify_base(self): else: real_shell = env + '="' + str(self.env_dict[env]) + '"\n' + real_shell self.stdio.verbose("real_shell: {0}".format(real_shell)) - process = subprocess.Popen(real_shell, shell=True, stdout=subprocess.PIPE) + process = subprocess.Popen(real_shell, shell=True, stdout=subprocess.PIPE, executable='/bin/bash') out, err = process.communicate() process.stdout.close() result = out[:-1].decode('utf-8') diff --git a/handler/checker/step/data_size.py b/handler/checker/step/data_size.py index dfb06d51..053d5068 100644 --- a/handler/checker/step/data_size.py +++ b/handler/checker/step/data_size.py @@ -17,9 +17,7 @@ """ from common.types import Capacity from handler.checker.check_exception import StepExecuteFailException -from common.ssh import SshHelper from handler.checker.check_report import TaskReport -from common.tool import Util class DataSizeHandler: @@ -27,16 +25,16 @@ def __init__(self, context, step, node, task_variable_dict): self.context = context self.stdio = context.stdio self.stdio.verbose("init DataSizeHandler") - self.ssh_helper = None + self.ssh_client = None self.parameters = None self.step = step self.node = node self.task_variable_dict = task_variable_dict try: - self.ssh_helper = self.node["ssher"] - if self.ssh_helper is None: - raise Exception("self.ssh_helper is None.") + self.ssh_client = self.node["ssher"] + if self.ssh_client is None: + raise Exception("self.ssh_client is None.") except Exception as e: self.stdio.error("DataSizeHandler ssh init fail . Please check the NODES conf Exception : {0} .".format(e)) raise Exception("DataSizeHandler ssh init fail . Please check the NODES conf Exception : {0} .".format(e)) diff --git a/handler/checker/step/get_system_parameter.py b/handler/checker/step/get_system_parameter.py index 040ee492..6a762579 100644 --- a/handler/checker/step/get_system_parameter.py +++ b/handler/checker/step/get_system_parameter.py @@ -17,7 +17,6 @@ """ from handler.checker.check_exception import StepExecuteFailException -from common.ssh import SshHelper from handler.checker.check_report import TaskReport from common.tool import Util @@ -27,16 +26,16 @@ def __init__(self, context, step, node, task_variable_dict): self.context = context self.stdio = context.stdio self.stdio.verbose("init GetSystemParameterHandler") - self.ssh_helper = None + self.ssh_client = None self.parameters = None self.step = step self.node = node self.task_variable_dict = task_variable_dict try: - self.ssh_helper = self.node["ssher"] - if self.ssh_helper is None: - raise Exception("self.ssh_helper is None.") + self.ssh_client = self.node["ssher"] + if self.ssh_client is None: + raise Exception("self.ssh_client is None.") except Exception as e: self.stdio.error("GetSystemParameterHandler ssh init fail . Please check the NODES conf Exception : {0} .".format(e)) raise Exception("GetSystemParameterHandler ssh init fail . Please check the NODES conf Exception : {0} .".format(e)) @@ -48,8 +47,8 @@ def __init__(self, context, step, node, task_variable_dict): def get_parameter(self, parameter_name): try: parameter_name = parameter_name.replace(".", "/") - parameter_value = self.ssh_helper.ssh_exec_cmd("cat /proc/sys/" + parameter_name).strip() - self.ssh_helper.ssh_close() + parameter_value = self.ssh_client.exec_cmd("cat /proc/sys/" + parameter_name).strip() + self.ssh_client.ssh_close() except Exception as e: self.stdio.warn("get {0} fail:{1} .please check, the parameter_value will be set -1".format(parameter_name, e)) parameter_value = str("-1") @@ -68,7 +67,7 @@ def execute(self): else: s = self.step["parameter"] # SystemParameter exist? - if self.ssh_helper.ssh_exec_cmd('find /proc/sys/ -name "{0}"'.format(s)) == "": + if self.ssh_client.exec_cmd('find /proc/sys/ -name "{0}"'.format(s)) == "": self.stdio.warn("{0} is not exist".format(self.step["parameter"])) if "result" in self.step and "set_value" in self.step["result"]: self.task_variable_dict[self.step["result"]["set_value"]] = "" diff --git a/handler/checker/step/ssh.py b/handler/checker/step/ssh.py index 3f7c6247..adbfb27e 100644 --- a/handler/checker/step/ssh.py +++ b/handler/checker/step/ssh.py @@ -31,9 +31,9 @@ def __init__(self, context, step, node, task_variable_dict): self.step = step self.node = node try: - self.ssh_helper = self.node["ssher"] - if self.ssh_helper is None: - raise Exception("self.ssh_helper is None.") + self.ssh_client = self.node["ssher"] + if self.ssh_client is None: + raise Exception("self.ssh_client is None.") except Exception as e: self.stdio.error("SshHandler init fail. Please check the NODES conf. node: {0}. Exception : {1} .".format(node, e)) raise Exception("SshHandler init fail. Please check the NODES conf node: {0} Exception : {1} .".format(node, e)) @@ -47,7 +47,7 @@ def execute(self): raise StepExecuteFailException("SshHandler execute ssh is not set") ssh_cmd = StringUtils.build_str_on_expr_by_dict(self.step["ssh"], self.task_variable_dict) self.stdio.verbose("step SshHandler execute :{0} ".format(ssh_cmd)) - ssh_report_value = self.ssh_helper.ssh_exec_cmd(ssh_cmd) + ssh_report_value = self.ssh_client.exec_cmd(ssh_cmd) if ssh_report_value is None: ssh_report_value = "" if len(ssh_report_value) > 0: @@ -60,7 +60,7 @@ def execute(self): self.stdio.error("ssh execute Exception:{0}".format(e).strip()) raise StepExecuteFailException("ssh execute Exception:{0}".format(e).strip()) finally: - self.ssh_helper.ssh_close() + self.ssh_client.ssh_close() self.stdio.verbose("step SshHandler ssh_report_value:{0}".format(ssh_report_value)) def update_step_variable_dict(self): diff --git a/handler/checker/step/stepbase.py b/handler/checker/step/stepbase.py index 897566f5..78461f29 100644 --- a/handler/checker/step/stepbase.py +++ b/handler/checker/step/stepbase.py @@ -23,6 +23,7 @@ from handler.checker.step.ssh import SshHandler from handler.checker.step.sql import StepSQLHandler import docker +from common.ssh_client.ssh import SshClient class StepBase(object): @@ -44,7 +45,8 @@ def execute(self, report): self.task_variable_dict["remote_ip"] = self.node["ip"] elif "ssh_type" in self.node and self.node["ssh_type"] == "docker": self.stdio.verbose("execute ssh_type is docker") - self.task_variable_dict["remote_ip"] = docker.from_env().containers.get(self.node["container_name"]).attrs['NetworkSettings']['Networks']['bridge']["IPAddress"] + ssh_client = SshClient(self.context, self.node) + self.task_variable_dict["remote_ip"] = ssh_client.get_ip() for node in self.node: self.task_variable_dict["remote_{0}".format(node)] = self.node[node] if "type" not in self.step: @@ -102,7 +104,7 @@ def execute(self, report): if self.step["type"] == "sql": report.add("[cluster:{0}] {1}".format(self.cluster.get("ob_cluster_name") or self.cluster.get("obproxy_cluster_name") or no_cluster_name_msg, resultException), level) else: - report.add("[{0}:{1}] {2}".format(self.node.get("ssh_type") or "", self.node.get("container_name") or self.task_variable_dict.get("remote_ip") or "", resultException), level) + report.add("[{0}] {1}".format(SshClient(self.context, self.node).get_name(), resultException), level) if level == "critical": raise StepResultFailException(resultException) raise StepResultFalseException(resultException) @@ -113,7 +115,7 @@ def execute(self, report): if self.step["type"] == "sql": report.add("[cluster:{0}] {1}".format(self.cluster.get("ob_cluster_name") or self.cluster.get("obproxy_cluster_name") or no_cluster_name_msg, resultFailException), "fail") else: - report.add("[{0}:{1}] {2}".format(self.node.get("ssh_type") or "", self.node.get("container_name") or self.task_variable_dict.get("remote_ip") or "", resultFailException), "fail") + report.add("[{0}] {1}".format(SshClient(self.context, self.node).get_name(), resultFailException), "fail") raise StepResultFailException(resultFailException) except Exception as e: diff --git a/handler/checker/tasks/observer/cluster/data_path_settings.yaml b/handler/checker/tasks/observer/cluster/data_path_settings.yaml index 51bf1407..53878e33 100644 --- a/handler/checker/tasks/observer/cluster/data_path_settings.yaml +++ b/handler/checker/tasks/observer/cluster/data_path_settings.yaml @@ -45,17 +45,9 @@ task: verify_type: equal verify: 0 err_msg: "the log_dir_path of disk size over 16TB ,the type must be xfs" - - type: ssh ssh: "df -Th #{data_dir_path} | grep '/' | awk '{print $2}'" result: set_value: file_system verify: '[ "${file_system}" == "xfs" ] || [ "${file_system}" == "ext4" ]' err_msg: "ip:#{remote_ip} ,data_dir_path: #{data_dir_path} file_system is not xfs or ext4." - - type: ssh - ssh: "df -Th #{remote_home_path} | grep '/' | awk '{print $2}'" - result: - set_value: file_system - verify: '[ "${file_system}" == "xfs" ] || [ "${file_system}" == "ext4" ]' - err_msg: "ip:#{remote_ip} ,home_path: #{remote_home_path} file_system is not xfs or ext4." - diff --git a/handler/checker/tasks/observer/column_storage/tenant_parameters.yaml b/handler/checker/tasks/observer/column_storage/tenant_parameters.yaml index 9cfe275e..6f70f0a5 100644 --- a/handler/checker/tasks/observer/column_storage/tenant_parameters.yaml +++ b/handler/checker/tasks/observer/column_storage/tenant_parameters.yaml @@ -38,13 +38,6 @@ task: verify: '[ -z "$tenant_ids" ]' report_type: warning err_msg: 'ob_sql_work_area_percentage is recommended 30. There are some tenant need change: #{tenant_ids}' - - type: sql - sql: 'SELECT GROUP_CONCAT(DISTINCT tenant_id) from oceanbase.`CDB_OB_SYS_VARIABLES` where name ="max_allowed_packet" and TENANT_ID>1000 and VALUE<=67108864;' - result: - set_value: tenant_ids - verify: '[ -z "$tenant_ids" ]' - report_type: warning - err_msg: 'max_allowed_packet is recommended 67108864. There are some tenant need change: #{tenant_ids}' - type: sql sql: 'SELECT GROUP_CONCAT(DISTINCT tenant_id) from oceanbase.CDB_OB_SYS_VARIABLES where name ="max_allowed_packet" and TENANT_ID>1000 and VALUE<=67108864;' result: diff --git a/handler/checker/tasks/observer/system/dependent_software.yaml b/handler/checker/tasks/observer/system/dependent_software.yaml index c99e66d5..d0029c82 100644 --- a/handler/checker/tasks/observer/system/dependent_software.yaml +++ b/handler/checker/tasks/observer/system/dependent_software.yaml @@ -1,13 +1,6 @@ info: 'To detect dependent software, refer to: https://www.oceanbase.com/docs/enterprise-oceanbase-ocp-cn-1000000000125643' task: - steps: - - type: ssh - ssh: "getenforce" - result: - set_value: SELinux_switch - report_type: warning - verify: "[ 'Disabled' == ${SELinux_switch} ]" - err_msg: 'SELinux need Disabled. Now , it is #{SELinux_switch}.' - type: ssh ssh: 'systemctl is-active --quiet firewalld && echo "Running" || echo "Not Running"' result: diff --git a/handler/checker/tasks/observer/system/getenforce.yaml b/handler/checker/tasks/observer/system/getenforce.yaml new file mode 100644 index 00000000..b34b61d9 --- /dev/null +++ b/handler/checker/tasks/observer/system/getenforce.yaml @@ -0,0 +1,17 @@ +info: 'check SELinux by getenforce. refer to: https://www.oceanbase.com/docs/enterprise-oceanbase-ocp-cn-1000000000125643' +task: + - steps: + - type: ssh + ssh: 'if command -v getenforce &>/dev/null; then echo "exist"; fi' + result: + set_value: getenforce_exist + report_type: execution + verify: '[ "exist" != "${getenforce_exist}" ]' + err_msg: 'Do not warning. getenforce is exist. We will check SELinux by getenforce' + - type: ssh + ssh: "getenforce" + result: + set_value: SELinux_switch + report_type: warning + verify: "[ 'Disabled' == ${SELinux_switch} ]" + err_msg: 'SELinux need Disabled. Now , it is #{SELinux_switch}.' \ No newline at end of file diff --git a/handler/checker/tasks/observer/system/parameter_tcp_wmem.yaml b/handler/checker/tasks/observer/system/parameter_tcp_wmem.yaml index 9ed4a2fc..b4377aad 100644 --- a/handler/checker/tasks/observer/system/parameter_tcp_wmem.yaml +++ b/handler/checker/tasks/observer/system/parameter_tcp_wmem.yaml @@ -30,4 +30,4 @@ task: report_type: warning verify_type: between verify: "[8388608,16777216]" - err_msg: 'net.ipv4.tcp_wmem_max : #{tcp_wmem_max}. recommended: 65536 ≤ max≤ 131072' + err_msg: 'net.ipv4.tcp_wmem_max : #{tcp_wmem_max}. recommended: 8388608 ≤ max≤ 16777216' diff --git a/handler/gather/gather_ash_report.py b/handler/gather/gather_ash_report.py index e64ecb54..f6aa955e 100644 --- a/handler/gather/gather_ash_report.py +++ b/handler/gather/gather_ash_report.py @@ -18,10 +18,9 @@ import datetime import os -from common.command import get_observer_version, get_observer_version_by_sql +from common.command import get_observer_version from common.ob_connector import OBConnector from common.obdiag_exception import OBDIAGFormatException, OBDIAGException -from common.ssh import SshHelper from common.tool import DirectoryUtil, TimeUtils, Util, StringUtils from stdio import SafeStdio from colorama import Fore, Style @@ -72,15 +71,11 @@ def handle(self): def version_check(self): observer_version = "" try: - observer_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) + observer_version = get_observer_version(self.context) except Exception as e: - if len(self.observer_nodes) > 0: - ssher = SshHelper(self.observer_nodes[0]["ip"], self.observer_nodes[0]["ssh_port"], self.observer_nodes[0]["ssh_username"], self.observer_nodes[0]["ssh_password"]) - observer_version = get_observer_version(True, ssher, self.observer_nodes[0]["home_path"], self.stdio) - else: - self.stdio.warn("RCAHandler Failed to get observer version:{0}".format(e)) + self.stdio.warn("RCAHandler Failed to get observer version:{0}".format(e)) + return False self.stdio.verbose("RCAHandler.init get observer version: {0}".format(observer_version)) - if not (observer_version == "4.0.0.0" or StringUtils.compare_versions_greater(observer_version, "4.0.0.0")): self.stdio.error("observer version: {0}, must greater than 4.0.0.0".format(observer_version)) return False diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index f97e9bd5..8bbbf412 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -23,8 +23,7 @@ from common.obdiag_exception import OBDIAGFormatException from common.constant import const from common.command import get_file_size, download_file, is_empty_dir, rm_rf_file, get_logfile_name_list, mkdir, delete_empty_file, zip_encrypt_dir, zip_dir -from common.ssh import SshHelper -from common.command import SshClient, LocalClient +from common.command import SshClient from common.tool import TimeUtils from common.tool import Util from common.tool import DirectoryUtil @@ -175,56 +174,44 @@ def __handle_from_node(self, pack_dir_this_command, node): resp = {"skip": False, "error": "", "zip_password": "", "gather_pack_path": ""} remote_ip = node.get("ip") if self.is_ssh else NetUtils.get_inner_ip() remote_user = node.get("ssh_username") - remote_password = node.get("ssh_password") - remote_port = node.get("ssh_port") - remote_private_key = node.get("ssh_key_file") remote_home_path = node.get("home_path") ssh_failed = False - self.stdio.verbose('Sending Collect Shell Command to node {0} ...'.format(remote_ip)) - if "ssh_type" in node and node["ssh_type"] == "docker": - local_store_dir = "{0}/docker_{1}".format(pack_dir_this_command, node["container_name"]) - else: - local_store_dir = "{0}/{1}".format(pack_dir_this_command, remote_ip) try: - ssh = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + ssh_client = SshClient(self.context, node) + local_store_dir = "{0}/{1}".format(pack_dir_this_command, ssh_client.get_name()) except Exception as e: - self.stdio.exception('ssh {0}@{1}: failed, Please check the {2}'.format(remote_user, remote_ip, self.config_path)) - ssh_failed = True - resp["skip"] = True - resp["error"] = "Please check the {0}".format(self.config_path) + raise e - if not ssh_failed: - # transform timestamp(in us) to yyyymmddhhmmss (filename_time style) - from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str)) - to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str)) - gather_dir_name = "ob_log_{0}_{1}_{2}".format(ssh.host_ip, from_datetime_timestamp, to_datetime_timestamp) - gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name) - mkdir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) + # transform timestamp(in us) to yyyymmddhhmmss (filename_time style) + from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str)) + to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str)) + gather_dir_name = "ob_log_{0}_{1}_{2}".format(ssh_client.get_name(), from_datetime_timestamp, to_datetime_timestamp) + gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name) + mkdir(ssh_client, gather_dir_full_path, self.stdio) - log_list, resp = self.__handle_log_list(ssh, node, resp) - if resp["skip"]: - return resp - if self.context.get_variable("gather_mode") == "trace_id_log": - self.__grep_log_until_empty(ssh_helper=ssh, home_path=remote_home_path, log_list=log_list, gather_path=gather_dir_full_path) - else: - for log_name in log_list: - self.__pharse_log(ssh_helper=ssh, log_name=log_name, home_path=remote_home_path, gather_path=gather_dir_full_path) - delete_empty_file(self.is_ssh, ssh, gather_dir_full_path, self.stdio) + log_list, resp = self.__handle_log_list(ssh_client, node, resp) + if resp["skip"]: + return resp + if self.context.get_variable("gather_mode") == "trace_id_log": + self.__grep_log_until_empty(ssh_client=ssh_client, home_path=remote_home_path, log_list=log_list, gather_path=gather_dir_full_path) + else: + for log_name in log_list: + self.__pharse_log(ssh_client=ssh_client, log_name=log_name, home_path=remote_home_path, gather_path=gather_dir_full_path) + delete_empty_file(ssh_client, gather_dir_full_path) + is_empty = is_empty_dir(ssh_client, gather_dir_full_path, self.stdio) + if is_empty: + resp["error"] = "Empty file" + resp["zip_password"] = "" + rm_rf_file(ssh_client, gather_dir_full_path, self.stdio) + else: + self.__handle_zip_file(ssh_client, resp, gather_dir_name, pack_dir_this_command) - is_empty = is_empty_dir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) - if is_empty: - resp["error"] = "Empty file" - resp["zip_password"] = "" - rm_rf_file(self.is_ssh, ssh, gather_dir_full_path, self.stdio) - else: - self.__handle_zip_file(node.get("ip"), ssh, resp, gather_dir_name, pack_dir_this_command) - ssh.ssh_close() return resp - def __grep_log_until_empty(self, ssh_helper, home_path, log_list, gather_path): + def __grep_log_until_empty(self, ssh_client, home_path, log_list, gather_path): """ 按时间顺序排序日志，从最新的时间（或者从设置的时间）开始往前找日志，直到grep的结果不为空，再直到grep的结果为空，则停止 - :param ssh_helper, home_path, log_list, gather_path + :param ssh_client, home_path, log_list, gather_path :return: """ log_type_list = ['observer', 'election', 'rootservice'] @@ -232,7 +219,7 @@ def __grep_log_until_empty(self, ssh_helper, home_path, log_list, gather_path): # 理论上只有上述三种日志，other_log_list应该为空 other_log_list = [log_name for log_name in log_list if not any(log_name.startswith(prefix) for prefix in log_type_list)] for log_name in other_log_list: - self.__pharse_log(ssh_helper=ssh_helper, log_name=log_name, home_path=home_path, gather_path=gather_path) + self.__pharse_log(ssh_client=ssh_client, log_name=log_name, home_path=home_path, gather_path=gather_path) # wf结尾的日志非全量日志，不排查 # 形如observer.log等日志不方便排序，暂时删除，在后续重新加上 @@ -246,17 +233,16 @@ def __grep_log_until_empty(self, ssh_helper, home_path, log_list, gather_path): cur_type_log_list.insert(0, f'{log_type}.log') has_res = False for log_name in cur_type_log_list: - is_empty = self.__grep_log(ssh_helper=ssh_helper, log_name=log_name, home_path=home_path, gather_path=gather_path) + is_empty = self.__grep_log(ssh_client=ssh_client, log_name=log_name, home_path=home_path, gather_path=gather_path) if not is_empty: has_res = True elif has_res: # 已有结果且grep结果为空，说明相关日志已集齐 break - def __grep_log(self, ssh_helper, home_path, log_name, gather_path): + def __grep_log(self, ssh_client, home_path, log_name, gather_path): """ 处理传入的日志文件，将满足条件的日志文件归集到一起，并返回grep结果是否为空 - :param ssh_helper, log_name, gather_path :return is_empty: """ log_path = os.path.join(home_path, "log") @@ -265,8 +251,8 @@ def __grep_log(self, ssh_helper, home_path, log_name, gather_path): find_file_cmd = "find {gather_path} -type f -name {log_name} ! -empty".format(gather_path=gather_path, log_name=log_name) self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) self.stdio.verbose("grep files, run cmd = [{0}]".format(find_file_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) - find_file_res = SshClient(self.stdio).run(ssh_helper, find_file_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) + ssh_client.exec_cmd(grep_cmd) + find_file_res = ssh_client.exec_cmd(find_file_cmd) return find_file_res == "" else: raise Exception("grep arg is none.") @@ -286,7 +272,7 @@ def __handle_log_list(self, ssh, node, resp): return log_list, resp return log_list, resp - def __get_log_name(self, ssh_helper, node): + def __get_log_name(self, ssh_client, node): """ 通过传入的from to的时间来过滤一遍文件列表，提取出初步满足要求的文件列表 :param ssh_helper: @@ -299,17 +285,17 @@ def __get_log_name(self, ssh_helper, node): else: get_oblog = "ls -1 -F %s/observer.log* %s/rootservice.log* %s/election.log* | awk -F '/' '{print $NF}'" % (log_path, log_path, log_path) log_name_list = [] - log_files = SshClient(self.stdio).run(ssh_helper, get_oblog) if self.is_ssh else LocalClient(self.stdio).run(get_oblog) + log_files = ssh_client.exec_cmd(get_oblog) if log_files: - log_name_list = get_logfile_name_list(self.is_ssh, ssh_helper, self.from_time_str, self.to_time_str, log_path, log_files, self.stdio) + log_name_list = get_logfile_name_list(ssh_client, self.from_time_str, self.to_time_str, log_path, log_files, self.stdio) else: self.stdio.error('Unable to find the log file. Please provide the correct home_path, the default is [/root/observer]') return log_name_list - def __pharse_log(self, ssh_helper, home_path, log_name, gather_path): + def __pharse_log(self, ssh_client, home_path, log_name, gather_path): """ 处理传入的日志文件，将满足条件的日志文件归集到一起 - :param ssh_helper, log_name, gather_path + :param ssh_client, log_name, gather_path :return: """ log_path = os.path.join(home_path, "log") @@ -329,38 +315,37 @@ def __pharse_log(self, ssh_helper, home_path, log_name, gather_path): grep_litter_cmd += "| grep -e '{0}'".format(grep_option) grep_cmd = "cat {log_dir}/{log_name} {grep_options} >> {gather_path}/{log_name} ".format(grep_options=grep_litter_cmd, gather_path=gather_path, log_name=log_name, log_dir=log_path) self.stdio.verbose('grep files, run cmd = [{0}]'.format(grep_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) + ssh_client.exec_cmd(grep_cmd) else: cp_cmd = "cp {log_dir}/{log_name} {gather_path}/{log_name} ".format(gather_path=gather_path, log_name=log_name, log_dir=log_path) self.stdio.verbose('copy files, run cmd = [{0}]'.format(cp_cmd)) - SshClient(self.stdio).run(ssh_helper, cp_cmd) if self.is_ssh else LocalClient(self.stdio).run(cp_cmd) + ssh_client.exec_cmd(cp_cmd) - def __handle_zip_file(self, ip, ssh, resp, gather_dir_name, pack_dir_this_command): + def __handle_zip_file(self, ssh_client, resp, gather_dir_name, pack_dir_this_command): zip_password = "" gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name) - self.stdio.start_loading('[ip: {0}] zip observer log start'.format(ip)) + self.stdio.start_loading('[ip: {0}] zip observer log start'.format(ssh_client.get_name())) if self.zip_encrypt: zip_password = Util.gen_password(16) - zip_encrypt_dir(self.is_ssh, ssh, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) + zip_encrypt_dir(ssh_client, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) else: - zip_dir(self.is_ssh, ssh, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) - self.stdio.stop_loading('[ip: {0}] zip observer log end'.format(ip)) + zip_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) + self.stdio.stop_loading('[{0}] zip observer log end'.format(ssh_client.get_name())) gather_package_dir = "{0}.zip".format(gather_dir_full_path) - - gather_log_file_size = get_file_size(self.is_ssh, ssh, gather_package_dir, self.stdio) - self.stdio.print(FileUtil.show_file_size_tabulate(ip, gather_log_file_size)) + gather_log_file_size = get_file_size(ssh_client, gather_package_dir, self.stdio) + self.stdio.print(FileUtil.show_file_size_tabulate(ssh_client, gather_log_file_size)) local_store_path = "" if int(gather_log_file_size) < self.file_size_limit: local_store_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name) - download_file(self.is_ssh, ssh, gather_package_dir, local_store_path, self.stdio) + download_file(ssh_client, gather_package_dir, local_store_path, self.stdio) resp["error"] = "" resp["zip_password"] = zip_password else: resp["error"] = "File too large" resp["zip_password"] = "" - rm_rf_file(self.is_ssh, ssh, gather_package_dir, self.stdio) + rm_rf_file(ssh_client, gather_package_dir, self.stdio) resp["gather_pack_path"] = local_store_path - self.stdio.verbose("Collect pack gathered from node {0}: stored in {1}".format(ip, gather_package_dir)) + self.stdio.verbose("Collect pack gathered from node {0}: stored in {1}".format(ssh_client.get_name(), gather_package_dir)) return resp @staticmethod diff --git a/handler/gather/gather_obadmin.py b/handler/gather/gather_obadmin.py index 99f0a969..39169fc5 100644 --- a/handler/gather/gather_obadmin.py +++ b/handler/gather/gather_obadmin.py @@ -25,7 +25,6 @@ from common.command import LocalClient, SshClient, is_empty_dir from handler.base_shell_handler import BaseShellHandler from common.command import download_file, rm_rf_file, get_file_size, zip_encrypt_dir, zip_dir, get_observer_version -from common.ssh import SshHelper from common.tool import TimeUtils from common.tool import StringUtils from common.tool import Util @@ -170,34 +169,35 @@ def __handle_from_node(self, local_stored_path, node): remote_dir_name = "clog_{0}_{1}".format(remote_ip, now_time) remote_dir_full_path = "/tmp/{0}".format(remote_dir_name) ssh_failed = False + ssh_client = None try: - ssh_helper = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + ssh_client = SshClient(self.context, node) except Exception as e: self.stdio.error("ssh {0}@{1}: failed, Please check the {2}".format(remote_user, remote_ip, self.config_path)) - ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) + return resp if not ssh_failed: mkdir_cmd = "mkdir -p {0}".format(remote_dir_full_path) - SshClient(self.stdio).run(ssh_helper, mkdir_cmd) if self.is_ssh else LocalClient(self.stdio).run(mkdir_cmd) - ob_version = get_observer_version(self.is_ssh, ssh_helper, node.get("home_path"), self.stdio) + ssh_client.exec_cmd(mkdir_cmd) + ob_version = get_observer_version(self.context) if (ob_version != "" and not StringUtils.compare_versions_lower(ob_version, const.MAX_OB_VERSION_SUPPORT_GATHER_OBADMIN, self.stdio)) or ob_version == "": self.stdio.verbose("This version {0} does not support gather clog/slog . The max supported version less than {1}".format(ob_version, const.MAX_OB_VERSION_SUPPORT_GATHER_OBADMIN)) resp["error"] = "{0} not support gather clog/slog".format(ob_version) resp["gather_pack_path"] = "{0}".format(local_stored_path) resp["zip_password"] = "" return resp - log_list, resp = self.__handle_log_list(ssh_helper, remote_ip, resp) + log_list, resp = self.__handle_log_list(ssh_client, remote_ip, resp) for slog in log_list: - self.__gather_log_info(ssh_helper, node, slog, remote_dir_full_path) + self.__gather_log_info(ssh_client, node, slog, remote_dir_full_path) - self.__mv_log(ssh_helper, remote_dir_full_path) - if is_empty_dir(self.is_ssh, ssh_helper, "/tmp/{0}".format(remote_dir_name), self.stdio): + self.__mv_log(ssh_client, remote_dir_full_path) + if is_empty_dir(ssh_client, "/tmp/{0}".format(remote_dir_name), self.stdio): resp["error"] = "gather failed, folder is empty" resp["zip_password"] = "" else: - resp = self.__handle_zip_file(remote_ip, ssh_helper, resp, remote_dir_name, local_stored_path) - rm_rf_file(self.is_ssh, ssh_helper, remote_dir_full_path, self.stdio) + resp = self.__handle_zip_file(remote_ip, ssh_client, resp, remote_dir_name) + rm_rf_file(ssh_client, remote_dir_full_path, self.stdio) return resp def __handle_log_list(self, ssh, node, resp): @@ -214,35 +214,35 @@ def __handle_log_list(self, ssh, node, resp): return log_list, resp return log_list, resp - def __handle_zip_file(self, ip, ssh, resp, gather_dir_name, pack_dir_this_command): + def __handle_zip_file(self, ssh_client, resp, gather_dir_name, pack_dir_this_command): zip_password = "" gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name) if self.zip_encrypt: zip_password = Util.gen_password(16) - zip_encrypt_dir(self.is_ssh, ssh, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) + zip_encrypt_dir(ssh_client, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) else: - zip_dir(self.is_ssh, ssh, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) + zip_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio) gather_package_dir = "{0}.zip".format(gather_dir_full_path) - gather_log_file_size = get_file_size(self.is_ssh, ssh, gather_package_dir, self.stdio) - self.stdio.print(FileUtil.show_file_size_tabulate(ip, gather_log_file_size, self.stdio)) + gather_log_file_size = get_file_size(ssh_client, gather_package_dir, self.stdio) + self.stdio.print(FileUtil.show_file_size_tabulate(ssh_client, gather_log_file_size, self.stdio)) local_path = "" if int(gather_log_file_size) < self.file_size_limit: - local_path = download_file(self.is_ssh, ssh, gather_package_dir, pack_dir_this_command, self.stdio) + local_path = download_file(ssh_client, gather_package_dir, pack_dir_this_command, self.stdio) resp["error"] = "" resp["zip_password"] = zip_password else: resp["error"] = "File too large" resp["zip_password"] = "" - rm_rf_file(self.is_ssh, ssh, gather_package_dir, self.stdio) + rm_rf_file(ssh_client, gather_package_dir, self.stdio) resp["gather_pack_path"] = local_path - self.stdio.verbose("Collect pack gathered from node {0}: stored in {1}".format(ip, gather_package_dir)) + self.stdio.verbose("Collect pack gathered from node {0}: stored in {1}".format(ssh_client.get_name(), gather_package_dir)) return resp - def __get_log_name(self, ssh_helper, node): + def __get_log_name(self, ssh_client, node): """ 通过传入的from to的时间来过滤一遍slog文件列表，提取出文件创建的时间 - :param ssh_helper: + :param ssh_client: :return: list """ slog_dir = os.path.join(node.get("data_dir"), "/slog") @@ -251,7 +251,7 @@ def __get_log_name(self, ssh_helper, node): get_log = "ls -l SLOG_DIR --time-style '+.%Y%m%d%H%M%S' | awk '{print $7,$6}'".replace("SLOG_DIR", slog_dir) else: get_log = "ls -l CLOG_DIR --time-style '+.%Y%m%d%H%M%S' | awk '{print $7,$6}'".replace("CLOG_DIR", clog_dir) - log_files = SshClient(self.stdio).run(ssh_helper, get_log) if self.is_ssh else LocalClient(self.stdio).run(get_log) + log_files = ssh_client.exec_cmd(get_log) log_name_list = [] for file_name in log_files.split('\n'): if file_name == "": @@ -265,12 +265,12 @@ def __get_log_name(self, ssh_helper, node): if (log_time > from_time) and (log_time < to_time): log_name_list.append(str(log_name_fields[0]).rstrip()) if len(log_name_list): - self.stdio.verbose("Find the qualified log file {0} on Server [{1}], " "wait for the next step".format(log_name_list, ssh_helper.get_name())) + self.stdio.verbose("Find the qualified log file {0} on Server [{1}], " "wait for the next step".format(log_name_list, ssh_client.get_name())) else: - self.stdio.warn("No found the qualified log file on Server [{0}]".format(ssh_helper.get_name())) + self.stdio.warn("No found the qualified log file on Server [{0}]".format(ssh_client.get_name())) return log_name_list - def __gather_log_info(self, ssh_helper, node, log_name, remote_dir): + def __gather_log_info(self, ssh_client, node, log_name, remote_dir): home_path = node.get("home_path") obadmin_install_dir = os.path.join(home_path, "/bin") if self.ob_admin_mode == "slog": @@ -283,15 +283,15 @@ def __gather_log_info(self, ssh_helper, node, log_name, remote_dir): clog_name=log_name, ) self.stdio.verbose("gather obadmin info, run cmd = [{0}]".format(cmd)) - SshClient(self.stdio).run(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + ssh_client.exec_cmd(cmd) - def __mv_log(self, ssh_helper, remote_dir): + def __mv_log(self, ssh_client, remote_dir): if self.ob_admin_mode == "slog": cmd = "cd {remote_dir} && mv ob_admin.log ob_admin_slog.log".format(remote_dir=remote_dir) else: cmd = "cd {remote_dir} && mv ob_admin.log ob_admin_clog.log".format(remote_dir=remote_dir) self.stdio.verbose("mv log info, run cmd = [{0}]".format(cmd)) - SshClient(self.stdio).run(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + ssh_client.exec_cmd(cmd) @staticmethod def __get_overall_summary(node_summary_tuple, mode, is_zip_encrypt): diff --git a/handler/gather/gather_obproxy_log.py b/handler/gather/gather_obproxy_log.py index 083277ef..265e6446 100644 --- a/handler/gather/gather_obproxy_log.py +++ b/handler/gather/gather_obproxy_log.py @@ -26,7 +26,6 @@ from common.command import LocalClient, SshClient from common.constant import const from common.command import get_file_size, download_file, is_empty_dir, get_logfile_name_list, mkdir, delete_empty_file, rm_rf_file, zip_encrypt_dir, zip_dir -from common.ssh import SshHelper from common.tool import Util from common.tool import DirectoryUtil from common.tool import FileUtil @@ -178,46 +177,44 @@ def __handle_from_node(self, node, pack_dir_this_command): resp = {"skip": False, "error": "", "zip_password": "", "gather_pack_path": ""} remote_ip = node.get("ip") if self.is_ssh else NetUtils.get_inner_ip() remote_user = node.get("ssh_username") - remote_password = node.get("ssh_password") - remote_port = node.get("ssh_port") - remote_private_key = node.get("ssh_key_file") remote_home_path = node.get("home_path") ssh_failed = False self.stdio.verbose("Sending Collect Shell Command to node {0} ...".format(remote_ip)) DirectoryUtil.mkdir(path=pack_dir_this_command, stdio=self.stdio) + ssh_client = None try: - ssh = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + ssh_client = SshClient(self.context, node) except Exception as e: self.stdio.exception("ssh {0}@{1}: failed, Please check the {2}".format(remote_user, remote_ip, self.config_path)) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) + return resp if not ssh_failed: from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str)) to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str)) - gather_dir_name = "obproxy_log_{0}_{1}_{2}".format(ssh.host_ip, from_datetime_timestamp, to_datetime_timestamp) + gather_dir_name = "obproxy_log_{0}_{1}_{2}".format(ssh_client.get_name(), from_datetime_timestamp, to_datetime_timestamp) gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name) - mkdir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) + mkdir(ssh_client, gather_dir_full_path, self.stdio) - log_list, resp = self.__handle_log_list(ssh, node, resp) + log_list, resp = self.__handle_log_list(ssh_client, node, resp) if resp["skip"]: return resp for log_name in log_list: - self.__pharse_log(ssh_helper=ssh, log_name=log_name, home_path=remote_home_path, gather_path=gather_dir_full_path) - delete_empty_file(self.is_ssh, ssh, gather_dir_full_path, self.stdio) + self.__pharse_log(ssh_client=ssh_client, log_name=log_name, home_path=remote_home_path, gather_path=gather_dir_full_path) + delete_empty_file(ssh_client, gather_dir_full_path, self.stdio) - is_empty = is_empty_dir(self.is_ssh, ssh, gather_dir_full_path, self.stdio) + is_empty = is_empty_dir(ssh_client, gather_dir_full_path, self.stdio) if is_empty: resp["error"] = "Empty file" resp["zip_password"] = "" - rm_rf_file(self.is_ssh, ssh, gather_dir_full_path, self.stdio) + rm_rf_file(ssh_client, gather_dir_full_path, self.stdio) else: - self.__handle_zip_file(remote_ip, ssh, resp, gather_dir_name, pack_dir_this_command) - ssh.ssh_close() + self.__handle_zip_file(ssh_client, resp, gather_dir_name, pack_dir_this_command) return resp - def __handle_log_list(self, ssh, node, resp): - log_list = self.__get_log_name(ssh, node) + def __handle_log_list(self, ssh_client, node, resp): + log_list = self.__get_log_name(ssh_client, node) ip = node.get("ip") if len(log_list) > self.file_number_limit: self.stdio.warn("{0} The number of log files is {1}, out of range (0,{2}], " "Please adjust the query limit".format(ip, len(log_list), self.file_number_limit)) @@ -231,7 +228,7 @@ def __handle_log_list(self, ssh, node, resp): return log_list, resp return log_list, resp - def __get_log_name(self, ssh_helper, node): + def __get_log_name(self, ssh_client, node): home_path = node.get("home_path") log_path = os.path.join(home_path, "log") if self.scope == "obproxy" or self.scope == "obproxy_stat" or self.scope == "obproxy_digest" or self.scope == "obproxy_limit" or self.scope == "obproxy_slow" or self.scope == "obproxy_diagnosis" or self.scope == "obproxy_error": @@ -246,21 +243,18 @@ def __get_log_name(self, ssh_helper, node): log_path, log_path, ) - if self.is_ssh: - log_files = SshClient(self.stdio).run(ssh_helper, get_obproxy_log).strip() - else: - log_files = LocalClient(self.stdio).run(get_obproxy_log).strip() + log_files = ssh_client.exec_cmd(get_obproxy_log) log_name_list = [] if log_files: - log_name_list = get_logfile_name_list(self.is_ssh, ssh_helper, self.from_time_str, self.to_time_str, log_path, log_files, self.stdio) + log_name_list = get_logfile_name_list(ssh_client, self.from_time_str, self.to_time_str, log_path, log_files, self.stdio) else: self.stdio.error("Unable to find the log file. Please provide the correct home_path config and check obproxy {0} log exist".format(log_path)) return log_name_list - def __pharse_log(self, ssh_helper, home_path, log_name, gather_path): + def __pharse_log(self, ssh_client, home_path, log_name, gather_path): """ 处理传入的日志文件，将满足条件的日志文件归集到一起 - :param ssh_helper, log_name, gather_path + :param ssh_client, log_name, gather_path :return: """ log_path = os.path.join(home_path, "log") @@ -281,36 +275,36 @@ def __pharse_log(self, ssh_helper, home_path, log_name, gather_path): grep_cmd = "cat {log_dir}/{log_name} {grep_args} >> {gather_path}/{log_name} ".format(grep_args=grep_litter_cmd, gather_path=gather_path, log_name=log_name, log_dir=log_path) self.stdio.verbose("grep files, run cmd = [{0}]".format(grep_cmd)) - SshClient(self.stdio).run(ssh_helper, grep_cmd) if self.is_ssh else LocalClient(self.stdio).run(grep_cmd) + ssh_client.exec_cmd(grep_cmd) else: cp_cmd = "cp {log_dir}/{log_name} {gather_path}/{log_name} ".format(gather_path=gather_path, log_name=log_name, log_dir=log_path) self.stdio.verbose("copy files, run cmd = [{0}]".format(cp_cmd)) - SshClient(self.stdio).run(ssh_helper, cp_cmd) if self.is_ssh else LocalClient(self.stdio).run(cp_cmd) + ssh_client.exec_cmd(cp_cmd) - def __handle_zip_file(self, ip, ssh, resp, gather_dir_name, pack_dir_this_command): + def __handle_zip_file(self, ssh_client, resp, gather_dir_name, pack_dir_this_command): zip_password = "" gather_dir_full_path = "{0}/{1}".format(self.gather_log_temporary_dir, gather_dir_name) if self.zip_encrypt: zip_password = Util.gen_password(16) - zip_encrypt_dir(self.is_ssh, ssh, zip_password, self.gather_log_temporary_dir, gather_dir_name, self.stdio) + zip_encrypt_dir(ssh_client, zip_password, self.gather_log_temporary_dir, gather_dir_name, self.stdio) else: - zip_dir(self.is_ssh, ssh, self.gather_log_temporary_dir, gather_dir_name, self.stdio) + zip_dir(ssh_client, self.gather_log_temporary_dir, gather_dir_name, self.stdio) gather_package_dir = "{0}.zip".format(gather_dir_full_path) - gather_log_file_size = get_file_size(self.is_ssh, ssh, gather_package_dir, self.stdio) - self.stdio.print(FileUtil.show_file_size_tabulate(ip, gather_log_file_size)) + gather_log_file_size = get_file_size(ssh_client, gather_package_dir, self.stdio) + self.stdio.print(FileUtil.show_file_size_tabulate(ssh_client, gather_log_file_size)) local_path = "" if int(gather_log_file_size) < self.file_size_limit: local_store_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name) - local_path = download_file(self.is_ssh, ssh, gather_package_dir, local_store_path, self.stdio) + local_path = download_file(ssh_client, gather_package_dir, local_store_path, self.stdio) resp["error"] = "" resp["zip_password"] = zip_password else: resp["error"] = "File too large" resp["zip_password"] = "" - rm_rf_file(self.is_ssh, ssh, gather_package_dir, self.stdio) + rm_rf_file(ssh_client, gather_package_dir, self.stdio) resp["gather_pack_path"] = local_path - self.stdio.verbose("Collect pack gathered from node {0}: stored in {1}".format(ip, gather_package_dir)) + self.stdio.verbose("Collect pack gathered from node {0}: stored in {1}".format(ssh_client.get_name(), gather_package_dir)) return resp @staticmethod diff --git a/handler/gather/gather_obstack2.py b/handler/gather/gather_obstack2.py index 4dcfb720..2178a51e 100644 --- a/handler/gather/gather_obstack2.py +++ b/handler/gather/gather_obstack2.py @@ -26,8 +26,6 @@ from common.constant import const from common.command import LocalClient, SshClient from handler.base_shell_handler import BaseShellHandler -from common.tool import Util -from common.ssh import SshHelper from common.tool import TimeUtils from common.tool import Util from common.tool import DirectoryUtil @@ -41,7 +39,6 @@ def __init__(self, context, gather_pack_dir='./', is_scene=False): super(GatherObstack2Handler, self).__init__() self.context = context self.stdio = context.stdio - self.is_ssh = True self.local_stored_path = gather_pack_dir self.remote_stored_path = None self.is_scene = is_scene @@ -99,15 +96,8 @@ def handle_from_node(node): file_size = os.path.getsize(resp["gather_pack_path"]) gather_tuples.append((node.get("ip"), False, resp["error"], file_size, int(time.time() - st), resp["gather_pack_path"])) - if self.is_ssh: - for node in self.nodes: - handle_from_node(node) - else: - local_ip = NetUtils.get_inner_ip() - node = self.nodes[0] - node["ip"] = local_ip - for node in self.nodes: - handle_from_node(node) + for node in self.nodes: + handle_from_node(node) summary_tuples = self.__get_overall_summary(gather_tuples) self.stdio.print(summary_tuples) @@ -117,133 +107,124 @@ def handle_from_node(node): def __handle_from_node(self, local_stored_path, node): resp = {"skip": False, "error": "", "gather_pack_path": ""} - remote_ip = node.get("ip") if self.is_ssh else NetUtils.get_inner_ip() + remote_ip = node.get("ip") remote_user = node.get("ssh_username") - remote_password = node.get("ssh_password") - remote_port = node.get("ssh_port") - remote_private_key = node.get("ssh_key_file") self.stdio.verbose("Sending Collect Shell Command to node {0} ...".format(remote_ip)) DirectoryUtil.mkdir(path=local_stored_path, stdio=self.stdio) now_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S') remote_dir_name = "obstack2_{0}_{1}".format(remote_ip, now_time) remote_dir_full_path = "/tmp/{0}".format(remote_dir_name) ssh_failed = False + ssh_client = None try: - ssh_helper = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + ssh_client = SshClient(self.context, node) except Exception as e: self.stdio.exception("ssh {0}@{1}: failed, Please check the {2}".format(remote_user, remote_ip, self.config_path)) - ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) - - if not ssh_failed: - if not is_support_arch(self.is_ssh, ssh_helper, self.stdio): - resp["error"] = "remote server {0} arch not support gather obstack".format(ssh_helper.get_name()) - return resp - mkdir(self.is_ssh, ssh_helper, remote_dir_full_path, self.stdio) - - # install and chmod obstack2 - ob_version = get_observer_version(self.is_ssh, ssh_helper, node.get("home_path"), self.stdio) - if not StringUtils.compare_versions_greater(ob_version, const.MIN_OB_VERSION_SUPPORT_GATHER_OBSTACK): - self.stdio.verbose("This version {0} does not support gather obstack . The minimum supported version is {1}".format(ob_version, const.MIN_OB_VERSION_SUPPORT_GATHER_OBSTACK)) - resp["error"] = "{0} not support gather obstack".format(ob_version) - resp["gather_pack_path"] = "{0}".format(local_stored_path) - return resp - is_need_install_obstack = self.__is_obstack_exists(self.is_ssh, ssh_helper) - if is_need_install_obstack: - self.stdio.verbose("There is no obstack2 on the host {0}. It needs to be installed. " "Please wait a moment ...".format(remote_ip)) - if getattr(sys, 'frozen', False): - absPath = os.path.dirname(sys.executable) - else: - absPath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - obstack2_local_stored_full_path = os.path.join(absPath, const.OBSTACK2_LOCAL_STORED_PATH) - upload_file(self.is_ssh, ssh_helper, obstack2_local_stored_full_path, const.OBSTACK2_DEFAULT_INSTALL_PATH, self.stdio) - self.stdio.verbose("Installation of obstack2 is completed and gather begins ...") - - self.__chmod_obstack2(self.is_ssh, ssh_helper) - # get observer_pid - observer_pid_list = get_observer_pid(self.is_ssh, ssh_helper, node.get("home_path"), self.stdio) - # gather obstack2 info - for observer_pid in observer_pid_list: - user = self.__get_observer_execute_user(ssh_helper, observer_pid) - self.__gather_obstack2_info(self.is_ssh, ssh_helper, user, observer_pid, remote_dir_name, node) - try: - self.stdio.start_loading('gather obstack info') - self.is_ready(ssh_helper, observer_pid, remote_dir_name) - self.stdio.stop_loading('gather obstack info sucess') - except: - self.stdio.stop_loading('gather info failed') - self.stdio.error("Gather obstack info on the host {0} observer pid {1}".format(remote_ip, observer_pid)) - delete_file_force(self.is_ssh, ssh_helper, "/tmp/{dir_name}/observer_{pid}_obstack.txt".format(dir_name=remote_dir_name, pid=observer_pid), self.stdio) - pass - if is_empty_dir(self.is_ssh, ssh_helper, "/tmp/{0}".format(remote_dir_name), self.stdio): - resp["error"] = "gather failed, folder is empty" - return resp - - zip_dir(self.is_ssh, ssh_helper, "/tmp", remote_dir_name, self.stdio) - remote_zip_file_path = "{0}.zip".format(remote_dir_full_path) - - file_size = get_file_size(self.is_ssh, ssh_helper, remote_zip_file_path, self.stdio) - remote_file_full_path = "{0}.zip".format(remote_dir_full_path) - if int(file_size) < self.file_size_limit: - local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) - download_file(self.is_ssh, ssh_helper, remote_file_full_path, local_file_path, self.stdio) - resp["error"] = "" + raise Exception("Please check the {0}".format(self.config_path)) + + if not is_support_arch(ssh_client): + resp["error"] = "remote server {0} arch not support gather obstack".format(ssh_client.get_name()) + return resp + mkdir(ssh_client, remote_dir_full_path) + # install and chmod obstack2 + ob_version = get_observer_version(self.context) + if not StringUtils.compare_versions_greater(ob_version, const.MIN_OB_VERSION_SUPPORT_GATHER_OBSTACK): + self.stdio.verbose("This version {0} does not support gather obstack . The minimum supported version is {1}".format(ob_version, const.MIN_OB_VERSION_SUPPORT_GATHER_OBSTACK)) + resp["error"] = "{0} not support gather obstack".format(ob_version) + resp["gather_pack_path"] = "{0}".format(local_stored_path) + return resp + is_need_install_obstack = self.__is_obstack_exists(ssh_client) + if is_need_install_obstack: + self.stdio.verbose("There is no obstack2 on the host {0}. It needs to be installed. " "Please wait a moment ...".format(remote_ip)) + if getattr(sys, 'frozen', False): + absPath = os.path.dirname(sys.executable) else: - resp["error"] = "File too large" - delete_file_force(self.is_ssh, ssh_helper, remote_file_full_path, self.stdio) - ssh_helper.ssh_close() - resp["gather_pack_path"] = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) + absPath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + obstack2_local_stored_full_path = os.path.join(absPath, const.OBSTACK2_LOCAL_STORED_PATH) + upload_file(ssh_client, obstack2_local_stored_full_path, const.OBSTACK2_DEFAULT_INSTALL_PATH, self.context.stdio) + self.stdio.verbose("Installation of obstack2 is completed and gather begins ...") + + self.__chmod_obstack2(ssh_client) + # get observer_pid + observer_pid_list = get_observer_pid(ssh_client, node.get("home_path"), self.stdio) + # gather obstack2 info + for observer_pid in observer_pid_list: + user = self.__get_observer_execute_user(ssh_client, observer_pid) + self.__gather_obstack2_info(ssh_client, user, observer_pid, remote_dir_name, node) + try: + self.stdio.start_loading('gather obstack info') + self.is_ready(ssh_client, observer_pid, remote_dir_name) + self.stdio.stop_loading('gather obstack info sucess') + except: + self.stdio.stop_loading('gather info failed') + self.stdio.error("Gather obstack info on the host {0} observer pid {1}".format(remote_ip, observer_pid)) + delete_file_force(ssh_client, "/tmp/{dir_name}/observer_{pid}_obstack.txt".format(dir_name=remote_dir_name, pid=observer_pid), self.stdio) + pass + if is_empty_dir(ssh_client, "/tmp/{0}".format(remote_dir_name), self.stdio): + resp["error"] = "gather failed, folder is empty" + return resp + + zip_dir(ssh_client, "/tmp", remote_dir_name, self.stdio) + remote_zip_file_path = "{0}.zip".format(remote_dir_full_path) + + file_size = get_file_size(ssh_client, remote_zip_file_path, self.stdio) + remote_file_full_path = "{0}.zip".format(remote_dir_full_path) + if int(file_size) < self.file_size_limit: + local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) + download_file(ssh_client, remote_file_full_path, local_file_path, self.stdio) + resp["error"] = "" + else: + resp["error"] = "File too large" + delete_file_force(ssh_client, remote_file_full_path, self.stdio) + resp["gather_pack_path"] = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) return resp @Util.retry(5, 2) - def is_ready(self, ssh_helper, pid, remote_dir_name): + def is_ready(self, ssh_client, pid, remote_dir_name): try: self.stdio.verbose("Check whether the directory /tmp/{dir_name} or " "file /tmp/{dir_name}/observer_{pid}_obstack.txt is empty".format(dir_name=remote_dir_name, pid=pid)) - is_empty_dir_res = is_empty_dir(self.is_ssh, ssh_helper, "/tmp/{0}".format(remote_dir_name), self.stdio) - is_empty_file_res = is_empty_file(self.is_ssh, ssh_helper, "/tmp/{dir_name}/observer_{pid}_obstack.txt".format(dir_name=remote_dir_name, pid=pid), self.stdio) + is_empty_dir_res = is_empty_dir(ssh_client, "/tmp/{0}".format(remote_dir_name), self.stdio) + is_empty_file_res = is_empty_file(ssh_client, "/tmp/{dir_name}/observer_{pid}_obstack.txt".format(dir_name=remote_dir_name, pid=pid), self.stdio) if is_empty_dir_res or is_empty_file_res: self.stdio.verbose( - "The server {host_ip} directory /tmp/{dir_name} or file /tmp/{dir_name}/observer_{pid}_obstack.txt" - " is empty, waiting for the collection to complete".format(host_ip=ssh_helper.get_name() if self.is_ssh else NetUtils.get_inner_ip(self.stdio), dir_name=remote_dir_name, pid=pid) + "The server {host_ip} directory /tmp/{dir_name} or file /tmp/{dir_name}/observer_{pid}_obstack.txt" " is empty, waiting for the collection to complete".format(host_ip=ssh_client.get_name(), dir_name=remote_dir_name, pid=pid) ) raise except Exception as e: raise e - def __chmod_obstack2(self, is_ssh, ssh_helper): + def __chmod_obstack2(self, ssh_client): cmd = "chmod a+x {file}".format(file=const.OBSTACK2_DEFAULT_INSTALL_PATH) - SshClient(self.stdio).run(ssh_helper, cmd) if is_ssh else LocalClient(self.stdio).run(cmd) + ssh_client.exec_cmd(cmd) - def __is_obstack_exists(self, is_ssh, ssh_helper): + def __is_obstack_exists(self, ssh_client): cmd = "test -e {file} && echo exists".format(file=const.OBSTACK2_DEFAULT_INSTALL_PATH) - stdout = SshClient(self.stdio).run(ssh_helper, cmd) if is_ssh else LocalClient(self.stdio).run(cmd) + stdout = ssh_client.exec_cmd(cmd)[0] if stdout == 'exists': return False else: return True - def __get_observer_execute_user(self, ssh_helper, pid): + def __get_observer_execute_user(self, ssh_client, pid): cmd = "ps -o ruser=userForLongName -e -o pid,ppid,c,stime,tty,time,cmd | grep observer | grep {0} | awk {1}".format(pid, "'{print $1}'") - stdout = SshClient(self.stdio).run(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + stdout = ssh_client.exec_cmd(cmd) user = stdout.splitlines()[0] self.stdio.verbose("get observer execute user, run cmd = [{0}], result:{1} ".format(cmd, user)) return user - def __gather_obstack2_info(self, is_ssh, ssh_helper, user, observer_pid, remote_gather_dir, node): + def __gather_obstack2_info(self, ssh_client, user, observer_pid, remote_gather_dir, node): cmd = "{obstack} {pid} > /tmp/{gather_dir}/observer_{pid}_obstack.txt".format(obstack=const.OBSTACK2_DEFAULT_INSTALL_PATH, pid=observer_pid, gather_dir=remote_gather_dir) - if is_ssh: - if user == ssh_helper.username: - self.stdio.verbose("gather obstack info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), cmd)) - SshClient(self.stdio).run_ignore_err(ssh_helper, cmd) - else: - ssh_helper_new = SshHelper(ssh_helper.host_ip, ssh_helper.username, ssh_helper.password, ssh_helper.ssh_port, ssh_helper.key_file, node) - chown_cmd = "chown {user} /tmp/{gather_dir}/".format(user=user, gather_dir=remote_gather_dir) - SshClient(self.stdio).run(ssh_helper_new, chown_cmd) - self.stdio.verbose("gather obstack info on server {0}, run cmd = [su {1}, {2}]".format(ssh_helper.get_name(), user, cmd)) - ssh_helper_new.ssh_invoke_shell_switch_user(user, cmd, 10) + if user == ssh_client.exec_cmd('whoami'): + self.stdio.verbose("gather obstack info on server {0}, run cmd = [{1}]".format(ssh_client.get_name(), cmd)) + ssh_client.exec_cmd(cmd) else: - LocalClient(self.stdio).run(cmd) + chown_cmd = "chown {user} /tmp/{gather_dir}/".format(user=user, gather_dir=remote_gather_dir) + ssh_client.exec_cmd(chown_cmd) + self.stdio.verbose("gather obstack info on server {0}, run cmd = [su {1}, {2}]".format(ssh_client.get_name(), user, cmd)) + ssh_client.ssh_invoke_shell_switch_user(user, cmd, 10) + ssh_client.exec_cmd("rm -rf /tmp/{0}".format(remote_gather_dir)) @staticmethod def __get_overall_summary(node_summary_tuple): diff --git a/handler/gather/gather_perf.py b/handler/gather/gather_perf.py index 43930f86..e9244425 100644 --- a/handler/gather/gather_perf.py +++ b/handler/gather/gather_perf.py @@ -25,7 +25,6 @@ from common.command import LocalClient, SshClient from common.constant import const from handler.base_shell_handler import BaseShellHandler -from common.ssh import SshHelper from common.tool import Util from common.tool import DirectoryUtil from common.tool import FileUtil @@ -128,75 +127,74 @@ def __handle_from_node(self, node, local_stored_path): remote_dir_name = "perf_{0}_{1}".format(node.get("ip"), now_time) remote_dir_full_path = "/tmp/{0}".format(remote_dir_name) ssh_failed = False + ssh_client = None try: - ssh_helper = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + ssh_client = SshClient(self.context, node) except Exception as e: self.stdio.exception("ssh {0}@{1}: failed, Please check the {2}".format(remote_user, remote_ip, self.config_path)) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) + return resp if not ssh_failed: - mkdir(self.is_ssh, ssh_helper, remote_dir_full_path, self.stdio) - - pid_observer_list = get_observer_pid(self.is_ssh, ssh_helper, node.get("home_path"), self.stdio) + mkdir(ssh_client, remote_dir_full_path, self.stdio) + pid_observer_list = get_observer_pid(ssh_client, node.get("home_path"), self.stdio) if len(pid_observer_list) == 0: resp["error"] = "can't find observer" return resp for pid_observer in pid_observer_list: if self.scope == "sample": - self.__gather_perf_sample(ssh_helper, remote_dir_full_path, pid_observer) + self.__gather_perf_sample(ssh_client, remote_dir_full_path, pid_observer) elif self.scope == "flame": - self.__gather_perf_flame(ssh_helper, remote_dir_full_path, pid_observer) + self.__gather_perf_flame(ssh_client, remote_dir_full_path, pid_observer) else: - self.__gather_perf_sample(ssh_helper, remote_dir_full_path, pid_observer) - self.__gather_perf_flame(ssh_helper, remote_dir_full_path, pid_observer) - self.__gather_top(ssh_helper, remote_dir_full_path, pid_observer) + self.__gather_perf_sample(ssh_client, remote_dir_full_path, pid_observer) + self.__gather_perf_flame(ssh_client, remote_dir_full_path, pid_observer) + self.__gather_top(ssh_client, remote_dir_full_path, pid_observer) - zip_dir(self.is_ssh, ssh_helper, "/tmp", remote_dir_name, self.stdio) + zip_dir(ssh_client, "/tmp", remote_dir_name, self.stdio) remote_file_full_path = "{0}.zip".format(remote_dir_full_path) - file_size = get_file_size(self.is_ssh, ssh_helper, remote_file_full_path, self.stdio) + file_size = get_file_size(ssh_client, remote_file_full_path, self.stdio) if int(file_size) < self.file_size_limit: local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) - download_file(self.is_ssh, ssh_helper, remote_file_full_path, local_file_path, self.stdio) + download_file(ssh_client, remote_file_full_path, local_file_path, self.stdio) resp["error"] = "" else: resp["error"] = "File too large" - delete_file_force(self.is_ssh, ssh_helper, remote_file_full_path, self.stdio) - ssh_helper.ssh_close() + delete_file_force(ssh_client, remote_file_full_path, self.stdio) resp["gather_pack_path"] = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) return resp - def __gather_perf_sample(self, ssh_helper, gather_path, pid_observer): + def __gather_perf_sample(self, ssh_client, gather_path, pid_observer): try: cmd = "cd {gather_path} && perf record -o sample.data -e cycles -c 100000000 -p {pid} -g -- sleep 20".format(gather_path=gather_path, pid=pid_observer) self.stdio.verbose("gather perf sample, run cmd = [{0}]".format(cmd)) - SshClient(self.stdio).run_ignore_err(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) - + ssh_client.exec_cmd(cmd) generate_data = "cd {gather_path} && perf script -i sample.data -F ip,sym -f > sample.viz".format(gather_path=gather_path) self.stdio.verbose("generate perf sample data, run cmd = [{0}]".format(generate_data)) - SshClient(self.stdio).run_ignore_err(ssh_helper, generate_data) if self.is_ssh else LocalClient(self.stdio).run(generate_data) + ssh_client.exec_cmd(generate_data) except: - self.stdio.error("generate perf sample data on server [{0}] failed".format(ssh_helper.get_name())) + self.stdio.error("generate perf sample data on server [{0}] failed".format(ssh_client.get_name())) - def __gather_perf_flame(self, ssh_helper, gather_path, pid_observer): + def __gather_perf_flame(self, ssh_client, gather_path, pid_observer): try: perf_cmd = "cd {gather_path} && perf record -o flame.data -F 99 -p {pid} -g -- sleep 20".format(gather_path=gather_path, pid=pid_observer) self.stdio.verbose("gather perf, run cmd = [{0}]".format(perf_cmd)) - SshClient(self.stdio).run_ignore_err(ssh_helper, perf_cmd) if self.is_ssh else LocalClient(self.stdio).run(perf_cmd) + ssh_client.exec_cmd(perf_cmd) generate_data = "cd {gather_path} && perf script -i flame.data > flame.viz".format(gather_path=gather_path) self.stdio.verbose("generate perf data, run cmd = [{0}]".format(generate_data)) - SshClient(self.stdio).run_ignore_err(ssh_helper, generate_data) if self.is_ssh else LocalClient(self.stdio).run(generate_data) + ssh_client.exec_cmd(generate_data) except: - self.stdio.error("generate perf data on server [{0}] failed".format(ssh_helper.get_name())) + self.stdio.error("generate perf data on server [{0}] failed".format(ssh_client.get_name())) - def __gather_top(self, ssh_helper, gather_path, pid_observer): + def __gather_top(self, ssh_client, gather_path, pid_observer): try: cmd = "cd {gather_path} && top -Hp {pid} -b -n 1 > top.txt".format(gather_path=gather_path, pid=pid_observer) self.stdio.verbose("gather top, run cmd = [{0}]".format(cmd)) - SshClient(self.stdio).run(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + ssh_client.exec_cmd(cmd) except: - self.stdio.error("gather top on server failed [{0}]".format(ssh_helper.get_name())) + self.stdio.error("gather top on server failed [{0}]".format(ssh_client.get_name())) @staticmethod def __get_overall_summary(node_summary_tuple): diff --git a/handler/gather/gather_scenes.py b/handler/gather/gather_scenes.py index b4f35d18..073c112a 100644 --- a/handler/gather/gather_scenes.py +++ b/handler/gather/gather_scenes.py @@ -25,7 +25,7 @@ from handler.gather.scenes.list import GatherScenesListHandler from common.tool import DirectoryUtil from common.tool import StringUtils -from common.scene import get_obproxy_and_ob_version +from common.scene import get_version_by_type from colorama import Fore, Style from common.tool import Util from common.tool import TimeUtils @@ -89,7 +89,7 @@ def __execute_yaml_task_one(self, task_name, task_data): try: self.stdio.print("execute tasks: {0}".format(task_name)) task_type = self.__get_task_type(task_name) - version = get_obproxy_and_ob_version(self.obproxy_nodes, self.ob_nodes, self.task_type, self.stdio) + version = get_version_by_type(self.context, task_type) if version: match = re.search(r'\d+(\.\d+){2}(?:\.\d+)?', version) if match: diff --git a/handler/gather/gather_sysstat.py b/handler/gather/gather_sysstat.py index 0f4d41a2..f9299c7d 100644 --- a/handler/gather/gather_sysstat.py +++ b/handler/gather/gather_sysstat.py @@ -24,7 +24,6 @@ from common.command import LocalClient, SshClient from common.command import get_file_size, download_file, mkdir, zip_dir from handler.base_shell_handler import BaseShellHandler -from common.ssh import SshHelper from common.tool import Util from common.tool import DirectoryUtil from common.tool import FileUtil @@ -120,126 +119,123 @@ def __handle_from_node(self, node, local_stored_path): resp = {"skip": False, "error": "", "gather_pack_path": ""} remote_ip = node.get("ip") if self.is_ssh else NetUtils.get_inner_ip() remote_user = node.get("ssh_username") - remote_password = node.get("ssh_password") - remote_port = node.get("ssh_port") - remote_private_key = node.get("ssh_key_file") self.stdio.verbose("Sending Collect Shell Command to node {0} ...".format(remote_ip)) DirectoryUtil.mkdir(path=local_stored_path, stdio=self.stdio) now_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S') remote_dir_name = "sysstat_{0}_{1}".format(remote_ip, now_time) remote_dir_full_path = "/tmp/{0}".format(remote_dir_name) ssh_failed = False + ssh_client = None try: - ssh_helper = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + ssh_client = SshClient(self.context, node) except Exception as e: self.stdio.exception("ssh {0}@{1}: failed, Please check the {2}".format(remote_user, remote_ip, self.config_path)) ssh_failed = True resp["skip"] = True resp["error"] = "Please check the {0}".format(self.config_path) if not ssh_failed: - mkdir(self.is_ssh, ssh_helper, remote_dir_full_path, self.stdio) - - self.__gather_dmesg_boot_info(ssh_helper, remote_dir_full_path) - self.__gather_dmesg_current_info(ssh_helper, remote_dir_full_path) - if self.__tsar_exit(ssh_helper): - self.__gather_cpu_info(ssh_helper, remote_dir_full_path) - self.__gather_mem_info(ssh_helper, remote_dir_full_path) - self.__gather_swap_info(ssh_helper, remote_dir_full_path) - self.__gather_io_info(ssh_helper, remote_dir_full_path) - self.__gather_traffic_info(ssh_helper, remote_dir_full_path) - self.__gather_tcp_udp_info(ssh_helper, remote_dir_full_path) - zip_dir(self.is_ssh, ssh_helper, "/tmp", remote_dir_name, self.stdio) + mkdir(ssh_client, remote_dir_full_path, self.stdio) + + self.__gather_dmesg_boot_info(ssh_client, remote_dir_full_path) + self.__gather_dmesg_current_info(ssh_client, remote_dir_full_path) + if self.__tsar_exit(ssh_client): + self.__gather_cpu_info(ssh_client, remote_dir_full_path) + self.__gather_mem_info(ssh_client, remote_dir_full_path) + self.__gather_swap_info(ssh_client, remote_dir_full_path) + self.__gather_io_info(ssh_client, remote_dir_full_path) + self.__gather_traffic_info(ssh_client, remote_dir_full_path) + self.__gather_tcp_udp_info(ssh_client, remote_dir_full_path) + zip_dir(ssh_client, "/tmp", remote_dir_name, self.stdio) remote_file_full_path = "{0}.zip".format(remote_dir_full_path) - file_size = get_file_size(self.is_ssh, ssh_helper, remote_file_full_path, self.stdio) + file_size = get_file_size(ssh_client, remote_file_full_path, self.stdio) if int(file_size) < self.file_size_limit: local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) self.stdio.verbose("local file path {0}...".format(local_file_path)) - download_file(self.is_ssh, ssh_helper, remote_file_full_path, local_file_path, self.stdio) + download_file(ssh_client, remote_file_full_path, local_file_path, self.stdio) resp["error"] = "" else: resp["error"] = "File too large" # delete_file_force(self.is_ssh, ssh_helper, remote_file_full_path) - ssh_helper.ssh_close() resp["gather_pack_path"] = "{0}/{1}.zip".format(local_stored_path, remote_dir_name) return resp - def __gather_dmesg_current_info(self, ssh_helper, gather_path): + def __gather_dmesg_current_info(self, ssh_client, gather_path): try: dmesg_cmd = "dmesg --ctime > {gather_path}/dmesg.human.current".format(gather_path=gather_path) self.stdio.verbose("gather dmesg current info, run cmd = [{0}]".format(dmesg_cmd)) - SshClient(self.stdio).run(ssh_helper, dmesg_cmd) if self.is_ssh else LocalClient(self.stdio).run(dmesg_cmd) + ssh_client.exec_cmd(dmesg_cmd) except: - self.stdio.error("Failed to gather dmesg current info on server {0}".format(ssh_helper.get_name())) + self.stdio.error("Failed to gather dmesg current info on server {0}".format(ssh_client.get_name())) - def __gather_dmesg_boot_info(self, ssh_helper, dir_path): + def __gather_dmesg_boot_info(self, ssh_client, dir_path): try: file_exit_cmd = "ls -l {file_path} 2>/dev/null".format(file_path="/var/log/dmesg") - file_exit = SshClient(self.stdio).run(ssh_helper, file_exit_cmd) if self.is_ssh else LocalClient(self.stdio).run(file_exit_cmd) + file_exit = ssh_client.exec_cmd(file_exit_cmd) if file_exit: dmesg_cmd = 'cp --force /var/log/dmesg {dir_path}/dmesg.boot'.format(dir_path=dir_path) - self.stdio.verbose("gather dmesg boot info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), dmesg_cmd)) - SshClient(self.stdio).run(ssh_helper, dmesg_cmd) if self.is_ssh else LocalClient(self.stdio).run(dmesg_cmd) + self.stdio.verbose("gather dmesg boot info on server {0}, run cmd = [{1}]".format(ssh_client.get_name(), dmesg_cmd)) + ssh_client.exec_cmd(dmesg_cmd) else: - self.stdio.warn("the file /var/log/dmesg on server {0} not found ".format(ssh_helper.get_name())) + self.stdio.warn("the file /var/log/dmesg on server {0} not found ".format(ssh_client.get_name())) except: - self.stdio.error("Failed to gather the /var/log/dmesg on server {0}".format(ssh_helper.get_name())) + self.stdio.error("Failed to gather the /var/log/dmesg on server {0}".format(ssh_client.get_name())) - def __tsar_exit(self, ssh_helper): + def __tsar_exit(self, ssh_client): try: cmd = "tar --help" - exit = SshClient(self.stdio).run(ssh_helper, cmd) if self.is_ssh else LocalClient(self.stdio).run(cmd) + exit = ssh_client.exec_cmd(cmd) if exit: return True except: self.stdio.warn("tsar not found") - def __gather_cpu_info(self, ssh_helper, gather_path): + def __gather_cpu_info(self, ssh_client, gather_path): try: tsar_cmd = "tsar --cpu -i 1 > {gather_path}/one_day_cpu_data.txt".format(gather_path=gather_path) - self.stdio.verbose("gather cpu info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + self.stdio.verbose("gather cpu info on server {0}, run cmd = [{1}]".format(ssh_client.get_name(), tsar_cmd)) + ssh_client.exec_cmd(tsar_cmd) except: - self.stdio.error("Failed to gather cpu info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error("Failed to gather cpu info use tsar on server {0}".format(ssh_client.get_name())) - def __gather_mem_info(self, ssh_helper, gather_path): + def __gather_mem_info(self, ssh_client, gather_path): try: tsar_cmd = "tsar --mem -i 1 > {gather_path}/one_day_mem_data.txt".format(gather_path=gather_path) - self.stdio.verbose("gather memory info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + self.stdio.verbose("gather memory info on server {0}, run cmd = [{1}]".format(ssh_client.get_name(), tsar_cmd)) + ssh_client.exec_cmd(tsar_cmd) except: - self.stdio.error("Failed to gather memory info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error("Failed to gather memory info use tsar on server {0}".format(ssh_client.get_name())) - def __gather_swap_info(self, ssh_helper, gather_path): + def __gather_swap_info(self, ssh_client, gather_path): try: tsar_cmd = "tsar --swap --load > {gather_path}/tsar_swap_data.txt".format(gather_path=gather_path) - self.stdio.verbose("gather swap info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + self.stdio.verbose("gather swap info on server {0}, run cmd = [{1}]".format(ssh_client.get_name(), tsar_cmd)) + ssh_client.exec_cmd(tsar_cmd) except: - self.stdio.error("Failed to gather swap info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error("Failed to gather swap info use tsar on server {0}".format(ssh_client.get_name())) - def __gather_io_info(self, ssh_helper, gather_path): + def __gather_io_info(self, ssh_client, gather_path): try: tsar_cmd = "tsar --io > {gather_path}/tsar_io_data.txt".format(gather_path=gather_path) - self.stdio.verbose("gather io info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + self.stdio.verbose("gather io info on server {0}, run cmd = [{1}]".format(ssh_client.get_name(), tsar_cmd)) + ssh_client.exec_cmd(tsar_cmd) except: - self.stdio.error("Failed to gather io info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error("Failed to gather io info use tsar on server {0}".format(ssh_client.get_name())) - def __gather_traffic_info(self, ssh_helper, gather_path): + def __gather_traffic_info(self, ssh_client, gather_path): try: tsar_cmd = "tsar --traffic > {gather_path}/tsar_traffic_data.txt".format(gather_path=gather_path) - self.stdio.verbose("gather traffic info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + self.stdio.verbose("gather traffic info on server {0}, run cmd = [{1}]".format(ssh_client.get_name(), tsar_cmd)) + ssh_client.exec_cmd(tsar_cmd) except: - self.stdio.error("Failed to gather traffic info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error("Failed to gather traffic info use tsar on server {0}".format(ssh_client.get_name())) - def __gather_tcp_udp_info(self, ssh_helper, gather_path): + def __gather_tcp_udp_info(self, ssh_client, gather_path): try: tsar_cmd = "tsar --tcp --udp -d 1 > {gather_path}/tsar_tcp_udp_data.txt".format(gather_path=gather_path) - self.stdio.verbose("gather tcp and udp info on server {0}, run cmd = [{1}]".format(ssh_helper.get_name(), tsar_cmd)) - SshClient(self.stdio).run(ssh_helper, tsar_cmd) if self.is_ssh else LocalClient(self.stdio).run(tsar_cmd) + self.stdio.verbose("gather tcp and udp info on server {0}, run cmd = [{1}]".format(ssh_client.get_name(), tsar_cmd)) + ssh_client.exec_cmd(tsar_cmd) except: - self.stdio.error("Failed to gather tcp and udp info use tsar on server {0}".format(ssh_helper.get_name())) + self.stdio.error("Failed to gather tcp and udp info use tsar on server {0}".format(ssh_client.get_name())) @staticmethod def __get_overall_summary(node_summary_tuple): diff --git a/handler/gather/scenes/cpu_high.py b/handler/gather/scenes/cpu_high.py index 4c6624d1..abf52d2e 100644 --- a/handler/gather/scenes/cpu_high.py +++ b/handler/gather/scenes/cpu_high.py @@ -16,8 +16,9 @@ @desc: """ import os + +from common.ssh_client.ssh import SshClient from stdio import SafeStdio -from common.ssh import SshHelper from handler.gather.gather_obstack2 import GatherObstack2Handler from handler.gather.gather_perf import GatherPerfHandler from handler.gather.gather_log import GatherLogHandler @@ -60,10 +61,10 @@ def __gather_current_clocksource(self): try: self.stdio.print("gather current_clocksource start") for node in self.nodes: - ssh_helper = SshHelper(self.is_ssh, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) + ssh_client = SshClient(self.context, node) cmd = 'cat /sys/devices/system/clocksource/clocksource0/current_clocksource' self.stdio.verbose("gather current_clocksource, run cmd = [{0}]".format(cmd)) - result = ssh_helper.ssh_exec_cmd(cmd) + result = ssh_client.exec_cmd(cmd) file_path = os.path.join(self.report_path, "current_clocksource_{ip}_result.txt".format(ip=str(node.get("ip")).replace('.', '_'))) self.report(file_path, cmd, result) self.stdio.print("gather current_clocksource end") diff --git a/handler/gather/scenes/px_collect_log.py b/handler/gather/scenes/px_collect_log.py index 1fbc1bec..437e04a7 100644 --- a/handler/gather/scenes/px_collect_log.py +++ b/handler/gather/scenes/px_collect_log.py @@ -15,9 +15,9 @@ @file: px_collect_log.py @desc: """ +from common.ssh_client.ssh import SshClient from handler.gather.gather_log import GatherLogHandler from common.command import uzip_dir_local, analyze_log_get_sqc_addr, delete_file_in_folder, find_home_path_by_port -from common.ssh import SshHelper import datetime @@ -112,14 +112,11 @@ def __find_home_path_by_port(self, ip_str, internal_port_str): if node["ip"] == ip_str: remote_ip = node.get("ip") remote_user = node.get("ssh_username") - remote_password = node.get("ssh_password") - remote_port = node.get("ssh_port") - remote_private_key = node.get("ssh_key_file") try: - ssh = SshHelper(self.is_ssh, remote_ip, remote_user, remote_password, remote_port, remote_private_key, node, self.stdio) + ssh_client = SshClient(self.context, node) + return find_home_path_by_port(ssh_client, internal_port_str, self.stdio) except Exception as e: self.stdio.error("ssh {0}@{1}: failed, Please check the config".format(remote_user, remote_ip)) - return find_home_path_by_port(True, ssh, internal_port_str, self.stdio) def parse_trace_id(self, trace_id): id_ = trace_id.split('-')[0].split('Y')[1] diff --git a/handler/gather/step/base.py b/handler/gather/step/base.py index 064014de..8729e35e 100644 --- a/handler/gather/step/base.py +++ b/handler/gather/step/base.py @@ -15,8 +15,8 @@ @file: base.py @desc: """ +from common.ssh_client.ssh import SshClient from stdio import SafeStdio -import docker from handler.gather.step.ssh import SshHandler from handler.gather.step.sql import StepSQLHandler from handler.gather.gather_log import GatherLogHandler @@ -47,7 +47,8 @@ def execute(self): self.task_variable_dict["remote_ip"] = self.node["ip"] elif "ssh_type" in self.node and self.node["ssh_type"] == "docker": self.stdio.verbose("execute ssh_type is docker") - self.task_variable_dict["remote_ip"] = docker.from_env().containers.get(self.node["container_name"]).attrs['NetworkSettings']['Networks']['bridge']["IPAddress"] + ssh_client = SshClient(self.context, self.node) + self.task_variable_dict["remote_ip"] = ssh_client.get_ip() self.task_variable_dict["remote_home_path"] = self.node["home_path"] if "type" not in self.step: diff --git a/handler/gather/step/ssh.py b/handler/gather/step/ssh.py index e2ea4c4e..cdc1511b 100644 --- a/handler/gather/step/ssh.py +++ b/handler/gather/step/ssh.py @@ -16,8 +16,9 @@ @desc: """ import os + +from common.ssh_client.ssh import SshClient from stdio import SafeStdio -from common.ssh import SshHelper from common.tool import StringUtils @@ -31,8 +32,7 @@ def __init__(self, context, step, node, report_path, task_variable_dict): self.node = node self.report_path = report_path try: - is_ssh = True - self.ssh_helper = SshHelper(is_ssh, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node) + self.ssh_client = SshClient(self.context, node) except Exception as e: self.stdio.error("SshHandler init fail. Please check the NODES conf. node: {0}. Exception : {1} .".format(node, e)) self.task_variable_dict = task_variable_dict @@ -46,7 +46,7 @@ def execute(self): return ssh_cmd = StringUtils.build_str_on_expr_by_dict_2(self.step["ssh"], self.task_variable_dict) self.stdio.verbose("step SshHandler execute :{0} ".format(ssh_cmd)) - ssh_report_value = self.ssh_helper.ssh_exec_cmd(ssh_cmd) + ssh_report_value = self.ssh_client.exec_cmd(ssh_cmd) if ssh_report_value is None: ssh_report_value = "" if len(ssh_report_value) > 0: @@ -55,7 +55,7 @@ def execute(self): except Exception as e: self.stdio.error("ssh execute Exception:{0}".format(e).strip()) finally: - self.ssh_helper.ssh_close() + self.ssh_client.ssh_close() self.stdio.verbose("gather step SshHandler ssh_report_value:{0}".format(self.ssh_report_value)) def update_step_variable_dict(self): diff --git a/handler/meta/sql_meta.py b/handler/meta/sql_meta.py index 922173ac..878210a5 100644 --- a/handler/meta/sql_meta.py +++ b/handler/meta/sql_meta.py @@ -1066,7 +1066,7 @@ def rm_value(self, key): table_id from oceanbase.gv$table where - tenant_id = ? + tenant_id = ##REPLACE_TENANT_ID## and database_id = ##REPLACE_DATABASE_ID## and table_name = '##REPLACE_TABLE_NAME' limit 1 @@ -1078,9 +1078,31 @@ def rm_value(self, key): ''' select t3.table_id as table_id - from (select con_id, owner, table_name, partitioned from oceanbase.CDB_TABLES) t1 - left join (select con_id, owner, object_name, object_id from oceanbase.CDB_OBJECTS where object_type = 'database') t2 ON t1.con_id =t2.con_id and t1.owner = t2.owner - left join (select con_id, owner, object_name, object_id as table_id from oceanbase.CDB_OBJECTS where object_type = 'table') t3 ON t1.con_id = t3.con_id and t1.owner = t3.owner and t1.table_name = t3.object_name + from + (select + con_id, + owner, + table_name, + partitioned + from oceanbase.CDB_TABLES) + t1 + left join + (select + con_id, + owner, + object_name, + object_id + from oceanbase.CDB_OBJECTS + where object_type = 'database' + ) t2 ON t1.con_id =t2.con_id and t1.owner = t2.owner + left join + (select + con_id, + owner, + object_name, + object_id as table_id + from oceanbase.CDB_OBJECTS where object_type = 'table' + ) t3 ON t1.con_id = t3.con_id and t1.owner = t3.owner and t1.table_name = t3.object_name where t1.con_id = ##REPLACE_CON_ID## and t2.object_id = ##REPLACE_OBJECT_ID## t1.table_name = ##REPLACE_TABLE_NAME## limit 1 ''', ) @@ -1313,7 +1335,7 @@ def rm_value(self, key): select table_schema databaseName, table_name tableName - from oceanbase.information_schema.tables + from information_schema.tables where table_schema = '##REPLACE_DATABASE_NAME##' and table_type='BASE TABLE' limit 1 ''', ) @@ -1344,7 +1366,7 @@ def rm_value(self, key): and a.table_id=b.table_id and a.column_id=b.column_id and b.column_name not like '%__substr%' - and a.tenant_id=? and a.table_id=? + and a.tenant_id=##REPLACE_TENANT_ID## and a.table_id=##REPLACE_TABLE_ID## ''', ) @@ -1364,7 +1386,7 @@ def rm_value(self, key): and a.table_id=b.table_id and a.column_id=b.column_id and b.column_name not like '%__substr%' - and a.tenant_id=? and a.table_id=? + and a.tenant_id=##REPLACE_TENANT_ID## and a.table_id=##REPLACE_TABLE_ID## GROUP BY b.column_name ''', ) @@ -1378,7 +1400,7 @@ def rm_value(self, key): des_hex_str(case max_value when '19070000FEFFFFFFFFFFFFFFFF01' then '0' else max_value end) `maxValue` from oceanbase.__all_column_statistic - where tenant_id = ? and table_id = ? + where tenant_id = ##REPLACE_TENANT_ID## and table_id = ##REPLACE_TABLE_ID## ''', ) @@ -1392,6 +1414,6 @@ def rm_value(self, key): from oceanbase.__all_virtual_column_statistic where - tenant_id = ? and table_id = ? + tenant_id = ##REPLACE_TENANT_ID## and table_id = ##REPLACE_TABLE_ID## ''', ) diff --git a/handler/rca/plugins/gather.py b/handler/rca/plugins/gather.py index 1ec13942..4a2ddec1 100644 --- a/handler/rca/plugins/gather.py +++ b/handler/rca/plugins/gather.py @@ -19,7 +19,6 @@ import zipfile from handler.gather.gather_log import GatherLogHandler -from common.tool import Util from handler.gather.gather_obproxy_log import GatherObProxyLogHandler diff --git a/handler/rca/rca_handler.py b/handler/rca/rca_handler.py index 42e11dcd..51da1a95 100644 --- a/handler/rca/rca_handler.py +++ b/handler/rca/rca_handler.py @@ -21,15 +21,14 @@ from textwrap import fill from common.command import ( get_obproxy_version, - get_observer_version_by_sql, get_observer_version, ) from prettytable import PrettyTable from common.ob_connector import OBConnector +from common.ssh_client.ssh import SshClient from handler.rca.plugins.gather import Gather_log from handler.rca.rca_exception import RCANotNeedExecuteException from handler.rca.rca_list import RcaScenesListHandler -from common.ssh import SshHelper from common.tool import Util from common.tool import StringUtils from colorama import Fore, Style @@ -46,15 +45,7 @@ def __init__(self, context): context_observer_nodes = [] if observer_nodes is not None: for node in observer_nodes: - ssh = SshHelper( - True, - node.get("ip"), - node.get("ssh_username"), - node.get("ssh_password"), - node.get("ssh_port"), - node.get("ssh_key_file"), - node, - ) + ssh = SshClient(context, node) node["ssher"] = ssh context_observer_nodes.append(node) self.context.set_variable("observer_nodes", context_observer_nodes) @@ -63,15 +54,7 @@ def __init__(self, context): context_obproxy_nodes = [] if obproxy_nodes is not None: for node in obproxy_nodes: - ssh = SshHelper( - True, - node.get("ip"), - node.get("ssh_username"), - node.get("ssh_password"), - node.get("ssh_port"), - node.get("ssh_key_file"), - node, - ) + ssh = SshClient(context, node) node["ssher"] = ssh context_obproxy_nodes.append(node) self.context.set_variable("obproxy_nodes", context_obproxy_nodes) @@ -101,17 +84,9 @@ def __init__(self, context): # build observer_version by sql or ssher. If using SSHer, the observer_version is set to node[0]. observer_version = "" try: - observer_version = get_observer_version_by_sql(self.ob_cluster, self.stdio) + observer_version = get_observer_version(self.context) except Exception as e: - if len(context_observer_nodes) > 0: - observer_version = get_observer_version( - True, - context_observer_nodes[0]["ssher"], - context_observer_nodes[0]["home_path"], - self.stdio, - ) - else: - self.stdio.warn("RCAHandler Failed to get observer version:{0}".format(e)) + self.stdio.warn("RCAHandler Failed to get observer version:{0}".format(e)) self.stdio.verbose("RCAHandler.init get observer version: {0}".format(observer_version)) if observer_version != "": self.stdio.verbose("RCAHandler.init get observer version: {0}".format(observer_version)) diff --git a/handler/rca/scene/disconnection_scene.py b/handler/rca/scene/disconnection_scene.py index c2c1e355..f7904e93 100644 --- a/handler/rca/scene/disconnection_scene.py +++ b/handler/rca/scene/disconnection_scene.py @@ -34,8 +34,8 @@ def init(self, context): if "home_path" not in node or len(node["home_path"].strip()) == 0: self.stdio.warn("obproxy_node home_path is empty") raise Exception("obproxy_node home_path is empty") - ssh_helper = node["ssher"] - if ssh_helper is None: + ssh_client = node["ssher"] + if ssh_client is None: raise Exception("obproxy_node:{0} ssher is None".format(node["ip"])) obproxy_version = self.obproxy_version if obproxy_version is None or len(obproxy_version.strip()) == 0: diff --git a/handler/rca/scene/major_hold_scene.py b/handler/rca/scene/major_hold_scene.py index 0eb890f8..e6170148 100644 --- a/handler/rca/scene/major_hold_scene.py +++ b/handler/rca/scene/major_hold_scene.py @@ -178,20 +178,20 @@ def execute(self): svr_ip = svrs[0][4] svr_port = svrs[0][5] node = None - ssh_helper = None + ssh_client = None for observer_node in self.observer_nodes: if observer_node["ip"] == svr_ip and observer_node["port"] == svr_port: node = observer_node - ssh_helper = observer_node["ssher"] + ssh_client = observer_node["ssher"] if node == None: self.stdio.error("can not find ls_svr by TENANT_ID:{2} ip:{0},port:{1}".format(svr_ip, svr_port, err_tenant_id)) break log_name = "/tmp/major_hold_scene_4_major_merge_progress_checker_{0}.log".format(err_tenant_id) - ssh_helper.ssh_exec_cmd('grep "major_merge_progress_checker" {0}/log/rootservice.log* | grep T{1} -m500 >{2}'.format(node.get("home_path"), err_tenant_id, log_name)) - ssh_helper.download(log_name, self.local_path) + ssh_client.exec_cmd('grep "major_merge_progress_checker" {0}/log/rootservice.log* | grep T{1} -m500 >{2}'.format(node.get("home_path"), err_tenant_id, log_name)) + ssh_client.download(log_name, self.local_path) tenant_record.add_record("download {0} to {1}".format(log_name, self.local_path)) - ssh_helper.ssh_exec_cmd("rm -rf {0}".format(log_name)) + ssh_client.exec_cmd("rm -rf {0}".format(log_name)) except Exception as e: self.stdio.error("MajorHoldScene execute 4 exception: {0}".format(e)) raise RCAExecuteException("MajorHoldScene execute 4 exception: {0}".format(e)) @@ -235,20 +235,20 @@ def diagnose_info_switch(self, sql_data, tenant_record): diagnose_info = sql_data[8] if "schedule medium failed" in diagnose_info: node = None - ssh_helper = None + ssh_client = None for observer_node in self.observer_nodes: if svr_ip == observer_node.get("ip"): node = observer_node - ssh_helper = observer_node["ssher"] + ssh_client = observer_node["ssher"] if node is None: raise RCAExecuteException("can not find observer node by ip:{0}, port:{1}".format(svr_ip, svr_port)) log_name = "/tmp/rca_major_hold_schedule_medium_failed_{1}_{2}_{0}.txt".format(tenant_id, svr_ip, svr_port) tenant_record.add_record("diagnose_info type is 'schedule medium failed'. time is {0},observer is {1}:{2},the log is {3}".format(create_time, svr_ip, svr_port, log_name)) - ssh_helper.ssh_exec_cmd('grep "schedule_medium_failed" {1}/log/observer.log* |grep -P "\[\d+\]" -m 1 -o >{0}'.format(log_name, node.get("home_path"))) - ssh_helper.download(log_name, local_path=self.local_path) + ssh_client.exec_cmd('grep "schedule_medium_failed" {1}/log/observer.log* |grep -P "\[\d+\]" -m 1 -o >{0}'.format(log_name, node.get("home_path"))) + ssh_client.download(log_name, local_path=self.local_path) tenant_record.add_record("download {0} to {1}".format(log_name, self.local_path)) - ssh_helper.ssh_exec_cmd("rm -rf {0}".format(log_name)) + ssh_client.exec_cmd("rm -rf {0}".format(log_name)) return elif "error_no=" in diagnose_info and "error_trace=" in diagnose_info: err_no = re.search("\berror_no=(\d+)\b", diagnose_info).group(1) @@ -280,37 +280,36 @@ def diagnose_info_switch(self, sql_data, tenant_record): ) ) node = None - ssh_helper = None + ssh_client = None for observer_node in self.observer_nodes: if svr_ip == observer_node.get("ip"): node = observer_node - ssh_helper = observer_node["ssher"] + ssh_client = observer_node["ssher"] if node is None: raise RCAExecuteException("can not find observer node by ip:{0}, port:{1}".format(svr_ip, svr_port)) log_name = "/tmp/rca_error_no_{1}_{2}_{0}.txt".format(tenant_id, svr_ip, svr_port) - ssh_helper.ssh_exec_cmd('grep "{0}" {1}/log/observer.log* >{2}'.format(err_trace, node.get("home_path"), log_name)) - ssh_helper.download(log_name, local_path=self.local_path) + ssh_client.exec_cmd('grep "{0}" {1}/log/observer.log* >{2}'.format(err_trace, node.get("home_path"), log_name)) + ssh_client.download(log_name, local_path=self.local_path) tenant_record.add_record("download {0} to {1}".format(log_name, self.local_path)) - ssh_helper.ssh_exec_cmd("rm -rf {0}".format(log_name)) + ssh_client.exec_cmd("rm -rf {0}".format(log_name)) node = None - ssh_helper = None + ssh_client = None for observer_node in self.observer_nodes: if svr_ip == observer_node.get("ip"): node = observer_node - ssh_helper = observer_node["ssher"] + ssh_client = observer_node["ssher"] if node is None: raise RCAExecuteException("can not find observer node by ip:{0}, port:{1}".format(svr_ip, svr_port)) tenant_record.add_record("diagnose_info type is 'error_no'. time is {0},observer is {1}:{2},the log is {3}".format(create_time, svr_ip, svr_port, log_name)) - ssh_helper.ssh_exec_cmd('cat observer.log* |grep "{1}" > /tmp/{0}'.format(log_name, err_trace)) - ssh_helper.download(log_name, local_path=self.local_path) + ssh_client.exec_cmd('cat observer.log* |grep "{1}" > /tmp/{0}'.format(log_name, err_trace)) + ssh_client.download(log_name, local_path=self.local_path) tenant_record.add_record("download {0} to {1}".format(log_name, self.local_path)) - ssh_helper.ssh_exec_cmd("rm -rf {0}".format(log_name)) + ssh_client.exec_cmd("rm -rf {0}".format(log_name)) return elif "weak read ts is not ready" in diagnose_info: cursor = self.ob_connector.execute_sql_return_cursor_dictionary("select * from oceanbase.__all_virtual_ls_info where tenant_id='{0}' and ls_id='{1}';".format(tenant_id, ls_id)) - columns = [column[0] for column in cursor.description] all_virtual_ls_info_data = cursor.fetchall() self.all_virtual_ls_info = json.dumps(all_virtual_ls_info_data, cls=DateTimeEncoder) tenant_record.add_record("sql:" + "select * from oceanbase.__all_virtual_ls_info where tenant_id='{0}' and ls_id='{1}';".format(tenant_id, ls_id) + "result:{0}".format(str(self.all_virtual_ls_info))) @@ -367,21 +366,20 @@ def diagnose_info_switch(self, sql_data, tenant_record): + "result:{0}".format(str(all_virtual_tablet_compaction_info)) ) node = None - ssh_helper = None + ssh_client = None for observer_node in self.observer_nodes: if svr_ip == observer_node.get("ip"): node = observer_node - ssh_helper = observer_node["ssher"] + ssh_client = observer_node["ssher"] if node is None: raise RCAExecuteException("can not find observer node by ip:{0}, port:{1}".format(svr_ip, svr_port)) - log_name = "/tmp/rca_major_hold_major_not_schedule_for_long_time_{1}_{2}_{0}.txt".format(create_time, svr_ip, svr_port) tenant_record.add_record("diagnose_info type is 'major not schedule for long time'. time is {0},observer is {1}:{2},the log is {3}".format(create_time, svr_ip, svr_port, log_name)) - thread_id = ssh_helper.ssh_exec_cmd('cat {0}/log/observer.log* |grep "MediumLoo" -m 1 |grep -P "\[\d+\]" -m 1 -o | grep -oP "\d+"'.format(node["home_path"], tenant_id)).strip() - ssh_helper.ssh_exec_cmd('cat {0}/log/observer.log | grep "{1}" -m 100> {2}'.format(node["home_path"], thread_id, log_name)) - ssh_helper.download(log_name, local_path=self.local_path) + thread_id = ssh_client.exec_cmd('cat {0}/log/observer.log* |grep "MediumLoo" -m 1 |grep -P "\[\d+\]" -m 1 -o | grep -oP "\d+"'.format(node["home_path"], tenant_id)).strip() + ssh_client.exec_cmd('cat {0}/log/observer.log | grep "{1}" -m 100> {2}'.format(node["home_path"], thread_id, log_name)) + ssh_client.download(log_name, local_path=self.local_path) tenant_record.add_record("download {0} to {1}".format(log_name, self.local_path)) - ssh_helper.ssh_exec_cmd("rm -rf {0}".format(log_name)) + ssh_client.exec_cmd("rm -rf {0}".format(log_name)) else: tenant_record.add_record("diagnose_info type is Unknown.") diff --git a/init.sh b/init.sh index adb9a90e..766b82d5 100755 --- a/init.sh +++ b/init.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash if [[ $# == 1 && $1 == "-f" ]]; then FORCE_DEPLOY="1" diff --git a/main.py b/main.py index 3fb4cf41..f3434a45 100644 --- a/main.py +++ b/main.py @@ -16,7 +16,7 @@ """ import sys -from cmd import MainCommand +from diag_cmd import MainCommand from stdio import IO ROOT_IO = IO(1) diff --git a/requirements3.txt b/requirements3.txt index ade61c6b..262f5eec 100644 --- a/requirements3.txt +++ b/requirements3.txt @@ -19,8 +19,8 @@ tabulate==0.8.9 zstandard==0.17.0 pycryptodome~=3.14.1 pick==1.2.0 -PyYAML==6.0 -prettytable==3.10.0 +PyYAML==6.0.2rc1 +prettytable==3.5.0 oyaml==1.0 xmltodict==0.13.0 subprocess32==3.5.4 @@ -36,3 +36,5 @@ halo==0.0.31 inspect2==0.1.2 sqlgpt-parser>=0.0.1a5 netifaces==0.11.0 +netifaces==0.11.0 +kubernetes==30.1.0 diff --git a/rpm/build.sh b/rpm/build.sh index d8fa1785..5ea86c23 100755 --- a/rpm/build.sh +++ b/rpm/build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash python_bin='python' W_DIR=`pwd` diff --git a/rpm/obdiag-build.sh b/rpm/obdiag-build.sh index 99be94e5..facb3e6b 100755 --- a/rpm/obdiag-build.sh +++ b/rpm/obdiag-build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash PROJECT_DIR=$1 PROJECT_NAME=$2