Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@Teingi
Teingi committed Jul 3, 2024
1 parent 8ec1bed commit 40683e5
Show file tree
Hide file tree
Showing 57 changed files with 2,116 additions and 1,105 deletions.
2 changes: 1 addition & 1 deletion README-CN.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ sh /usr/local/oceanbase-diagnostic-tool/init.sh

```shell
pip3 install -r requirements3.txt
sh dev_init.sh
./dev_init.sh
source ~/.bashrc
```

Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ To install obdiag on Python >= 3.8, run these commands:

```shell
pip3 install -r requirements3.txt
sh dev_init.sh
./dev_init.sh
source ~/.bashrc
```

Expand Down
303 changes: 118 additions & 185 deletions common/command.py
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions common/config_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import time
from collections import OrderedDict

from common.command import get_observer_version_by_sql
from common.command import get_observer_version
from common.constant import const
from common.ob_connector import OBConnector
from common.tool import DirectoryUtil
Expand Down Expand Up @@ -51,7 +51,7 @@ def __init__(self, context):
}

def get_cluster_name(self):
ob_version = get_observer_version_by_sql(self.ob_cluster, self.stdio)
ob_version = get_observer_version(self.context)
obConnetcor = OBConnector(ip=self.db_host, port=self.db_port, username=self.sys_tenant_user, password=self.sys_tenant_password, stdio=self.stdio, timeout=100)
if ob_version.startswith("3") or ob_version.startswith("2"):
sql = "select cluster_name from oceanbase.v$ob_cluster"
Expand All @@ -64,7 +64,7 @@ def get_cluster_name(self):
return "obcluster"

def get_host_info_list_by_cluster(self):
ob_version = get_observer_version_by_sql(self.ob_cluster, self.stdio)
ob_version = get_observer_version(self.context)
obConnetcor = OBConnector(ip=self.db_host, port=self.db_port, username=self.sys_tenant_user, password=self.sys_tenant_password, stdio=self.stdio, timeout=100)
sql = "select SVR_IP, SVR_PORT, ZONE, BUILD_VERSION from oceanbase.DBA_OB_SERVERS"
if ob_version.startswith("3") or ob_version.startswith("2") or ob_version.startswith("1"):
Expand Down
46 changes: 11 additions & 35 deletions common/scene.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@
@file: scene.py
@desc:
"""
from common.ssh import SshHelper
from common.tool import StringUtils
from common.command import get_observer_version, get_obproxy_version, get_observer_version_by_sql
from common.command import get_observer_version, get_obproxy_version


def filter_by_version(scene, cluster, stdio=None):
Expand Down Expand Up @@ -66,44 +65,21 @@ def filter_by_version(scene, cluster, stdio=None):
raise Exception("filter_by_version Exception : {0}".format(e))


def get_version(nodes, type, cluster, stdio=None):
def get_version_by_type(context, type, stdio=None):
try:
if len(nodes) < 1:
raise Exception("input nodes is empty, please check your config")
node = nodes[0]
ssh = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node)
stdio = context.stdio
version = ""
if type == "observer":
if type == "observer" or type == "other":
try:
version = get_observer_version_by_sql(cluster, stdio)
version = get_observer_version(context)
except Exception as e:
stdio.warn("get observer version by sql fail, use node ssher to get. Exception:{0}".format(e))
version = get_observer_version(True, ssh, nodes[0]["home_path"], stdio)
elif type == "obproxy":
version = get_obproxy_version(True, ssh, nodes[0]["home_path"], stdio)
return version
except Exception as e:
stdio.exception("can't get version, Exception: {0}".format(e))
raise Exception("can't get version, Exception: {0}".format(e))


def get_obproxy_and_ob_version(obproxy_nodes, nodes, type, stdio=None):
try:
if type == "observer" or type == "other":
if len(nodes) < 1:
raise Exception("input nodes is empty, please check your config")
node = nodes[0]
ssh = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node)
version = get_observer_version(True, ssh, nodes[0]["home_path"], stdio)
stdio.warn("get observer version fail . Exception:{0}".format(e))
raise Exception("can't get observer version, Exception: {0}".format(e))
elif type == "obproxy":
if len(nodes) < 1:
raise Exception("input obproxy nodes is empty, please check your config")
node = obproxy_nodes[0]
ssh = SshHelper(True, node.get("ip"), node.get("ssh_username"), node.get("ssh_password"), node.get("ssh_port"), node.get("ssh_key_file"), node)
version = get_obproxy_version(True, ssh, nodes[0]["home_path"], stdio)
version = get_obproxy_version(context)
else:
raise Exception("type is {0} . No func to get the version".format(type))
raise Exception("type is {0} . No support to get the version".format(type))
return version
except Exception as e:
stdio.exception("can't get version, Exception: {0}".format(e))
raise Exception("can't get version, Exception: {0}".format(e))
stdio.exception("can't get {0} version, Exception: {1}".format(type, e))
raise Exception("can't get {0} version, Exception: {1}".format(type, e))
277 changes: 0 additions & 277 deletions common/ssh.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -802,280 +802,3 @@ def file_uploader(self, local_dir, remote_dir, stdio=None):
except:
stdio.exception("")
stdio.verbose('Failed to get %s' % remote_dir)


# TODO ENV_DISABLE_RSA_ALGORITHMS need get by context.inner_context
ENV_DISABLE_RSA_ALGORITHMS = 0


def dis_rsa_algorithms(state=0):
"""
Disable RSA algorithms in OpenSSH server.
"""
global ENV_DISABLE_RSA_ALGORITHMS
ENV_DISABLE_RSA_ALGORITHMS = state


class SshHelper(object):
def __init__(self, is_ssh=None, host_ip=None, username=None, password=None, ssh_port=None, key_file=None, node=None, stdio=None):
if node is None:
node = {}
self.is_ssh = is_ssh
self.stdio = stdio
self.host_ip = host_ip
self.username = username
self.ssh_port = node.get("ssh_port") or ssh_port
self.need_password = True
self.password = node.get("ssh_password") or password
self.key_file = node.get("ssh_key_file") or key_file
self.key_file = os.path.expanduser(self.key_file)
self.ssh_type = node.get("ssh_type") or "remote"
self._ssh_fd = None
self._sftp_client = None
if "ssh_type" in node and node.get("ssh_type") == "docker":
try:
self.ssh_type = node["ssh_type"]
self.stdio.verbose("use ssh_type:{0} , node info : {1}".format(self.ssh_type, StringUtils.node_cut_passwd_for_log(node)))
self.node = node
# docker_permissions_check
if self.ssh_type == "docker":
self.client = docker.from_env()
if "container_name" not in node:
self.stdio.error("SshHelper init docker Exception: 'container_name' not in node")
raise Exception("SshHelper init docker Exception: 'container_name' not in node")
else:
self.stdio.error("SshHelper init not support the ssh_type : {0}".format(self.ssh_type))
raise Exception("SshHelper init not support the ssh_type : {0}".format(self.ssh_type))

except Exception as e:
self.stdio.error("SshHelper init docker Exception: {0}".format(e))
raise Exception("SshHelper init docker Exception: {0}".format(e))

return

if self.is_ssh:
self._disabled_rsa_algorithms = None
DISABLED_ALGORITHMS = dict(pubkeys=["rsa-sha2-512", "rsa-sha2-256"])
if ENV_DISABLE_RSA_ALGORITHMS == 1:
self._disabled_rsa_algorithms = DISABLED_ALGORITHMS
self.ssh_type = "remote"
if len(self.key_file) > 0:
try:
self._ssh_fd = paramiko.SSHClient()
self._ssh_fd.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
self._ssh_fd.load_system_host_keys()
self._ssh_fd.connect(hostname=host_ip, username=username, key_filename=self.key_file, port=ssh_port, disabled_algorithms=self._disabled_rsa_algorithms)
except AuthenticationException:
self.password = input("Authentication failed, Input {0}@{1} password:\n".format(username, host_ip))
self.need_password = True
self._ssh_fd.connect(hostname=host_ip, username=username, password=password, port=ssh_port, disabled_algorithms=self._disabled_rsa_algorithms)
except Exception as e:
raise OBDIAGSSHConnException("ssh {0}@{1}: failed, exception:{2}".format(username, host_ip, e))
else:
self._ssh_fd = paramiko.SSHClient()
self._ssh_fd.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
self._ssh_fd.load_system_host_keys()
self.need_password = True
self._ssh_fd.connect(hostname=host_ip, username=username, password=password, port=ssh_port, disabled_algorithms=self._disabled_rsa_algorithms)

def ssh_exec_cmd(self, cmd):
if self.ssh_type == "docker":
try:
self.stdio.verbose("ssh_exec_cmd docker {0} cmd: {1}".format(self.node.get("container_name"), cmd))
client_result = self.client.containers.get(self.node["container_name"])
result = client_result.exec_run(
cmd=["bash", "-c", cmd],
detach=False,
stdout=True,
stderr=True,
)
if result.exit_code != 0:
raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.node["container_name"], cmd, result.output.decode('utf-8')))

except Exception as e:
self.stdio.error("sshHelper ssh_exec_cmd docker Exception: {0}".format(e))
raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e))

return result.output.decode('utf-8')
try:
stdin, stdout, stderr = self._ssh_fd.exec_command(cmd)
err_text = stderr.read()
if len(err_text):
raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.host_ip, cmd, err_text))
except SSHException as e:
raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.host_ip, cmd, e))
return stdout.read().decode('utf-8')

def ssh_exec_cmd_ignore_err(self, cmd):
if self.ssh_type == "docker":
try:
client_result = self.client.containers.get(self.node["container_name"])
result = client_result.exec_run(
cmd=["bash", "-c", cmd],
detach=False,
stdout=True,
stderr=True,
)
except Exception as e:
self.stdio.error("sshHelper ssh_exec_cmd docker Exception: {0}".format(e))
raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e))

return result.output.decode('utf-8')

try:
stdin, stdout, stderr = self._ssh_fd.exec_command(cmd)
return stdout.read().decode('utf-8')
except SSHException as e:
print("Execute Shell command on server {0} failed,command=[{1}], exception:{2}".format(self.node, cmd, e))

def ssh_exec_cmd_ignore_exception(self, cmd):
if self.ssh_type == "docker":
try:
client_result = self.client.containers.get(self.node["container_name"])
result = client_result.exec_run(
cmd=["bash", "-c", cmd],
detach=False,
stdout=True,
stderr=True,
)
return result.output.decode('utf-8')
except Exception as e:
self.stdio.error("sshHelper ssh_exec_cmd_ignore_exception docker Exception: {0}".format(e))
pass
# raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e))
return

try:
stdin, stdout, stderr = self._ssh_fd.exec_command(cmd)
return stderr.read().decode('utf-8')
except SSHException as e:
pass

def ssh_exec_cmd_get_stderr(self, cmd):
if self.ssh_type == "docker":
try:
client_result = self.client.containers.get(self.node["container_name"])
result = client_result.exec_run(
cmd=["bash", "-c", cmd],
detach=False,
stdout=True,
stderr=True,
)
return result.output.decode('utf-8')
except Exception as e:
self.stdio.error("sshHelper ssh_exec_cmd_ignore_exception docker Exception: {0}".format(e))
pass
# raise Exception("sshHelper ssh_exec_cmd docker Exception: {0}".format(e))
return
try:
stdin, stdout, stderr = self._ssh_fd.exec_command(cmd)
return stderr.read().decode('utf-8')
except SSHException as e:
pass

def progress_bar(self, transferred, to_be_transferred, suffix=''):
bar_len = 20
filled_len = int(round(bar_len * transferred / float(to_be_transferred)))
percents = round(20.0 * transferred / float(to_be_transferred), 1)
bar = '\033[32;1m%s\033[0m' % '=' * filled_len + '-' * (bar_len - filled_len)
print_percents = round((percents * 5), 1)
sys.stdout.flush()
sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix))
if transferred == to_be_transferred:
sys.stdout.write('Downloading [%s] %s%s%s %s %s\r' % (bar, '\033[32;1m%s\033[0m' % print_percents, '% [', self.translate_byte(transferred), ']', suffix))
print()

def download(self, remote_path, local_path):
if self.ssh_type == "docker":
try:
self.stdio.verbose("remote_path: {0}:{1} to local_path:{2}".format(self.node["container_name"], remote_path, local_path))
client_result = self.client.containers.get(self.node["container_name"])
data, stat = client_result.get_archive(remote_path)
with open(local_path, "wb") as f:
for chunk in data:
f.write(chunk)
return
except Exception as e:
self.stdio.error("sshHelper download docker Exception: {0}".format(e))
raise Exception("sshHelper download docker Exception: {0}".format(e))
return

transport = self._ssh_fd.get_transport()
self._sftp_client = paramiko.SFTPClient.from_transport(transport)
print('Download {0}:{1}'.format(self.host_ip, remote_path))
self._sftp_client.get(remote_path, local_path, callback=self.progress_bar)
self._sftp_client.close()

def translate_byte(self, B):
B = float(B)
KB = float(1024)
MB = float(KB**2)
GB = float(MB**2)
TB = float(GB**2)
if B < KB:
return '{} {}'.format(B, 'bytes' if B > 1 else "byte")
elif KB < B < MB:
return '{:.2f} KB'.format(B / KB)
elif MB < B < GB:
return '{:.2f} MB'.format(B / MB)
elif GB < B < TB:
return '{:.2f} GB'.format(B / GB)
else:
return '{:.2f} TB'.format(B / TB)

def upload(self, remote_path, local_path):
if self.ssh_type == "docker":
try:
self.stdio.verbose(" local_path:{0} to remote_path:{1}:{2}".format(local_path, self.node["container_name"], remote_path))

self.client.containers.get(self.node["container_name"]).put_archive(remote_path, local_path)

return
except Exception as e:
self.stdio.error("sshHelper upload docker Exception: {0}".format(e))
raise Exception("sshHelper upload docker Exception: {0}".format(e))
return
transport = self._ssh_fd.get_transport()
self._sftp_client = paramiko.SFTPClient.from_transport(transport)
self._sftp_client.put(remote_path, local_path)
self._sftp_client.close()

def ssh_close(self):
if self.ssh_type == "docker":
self.client.close()
return
if self._sftp_client is not None:
self._sftp_client.close()
self._sftp_client = None

def __del__(self):
if self._sftp_client is not None:
self._sftp_client.close()
self._sftp_client = None

def ssh_invoke_shell_switch_user(self, new_user, cmd, time_out):
if self.ssh_type == "docker":
try:
exec_id = self.client.exec_create(container=self.node["container_name"], command=['su', '- ' + new_user])
response = self.client.exec_start(exec_id)

return response
except Exception as e:
self.stdio.error("sshHelper ssh_invoke_shell_switch_user docker Exception: {0}".format(e))
raise Exception("sshHelper ssh_invoke_shell_switch_user docker Exception: {0}".format(e))
return
try:
ssh = self._ssh_fd.invoke_shell()
ssh.send('su {0}\n'.format(new_user))
ssh.send('{}\n'.format(cmd))
time.sleep(time_out)
self._ssh_fd.close()
result = ssh.recv(65535)
except SSHException as e:
raise OBDIAGShellCmdException("Execute Shell command on server {0} failed, " "command=[{1}], exception:{2}".format(self.host_ip, cmd, e))
return result

def get_name(self):
if self.ssh_type == "docker":
return "(docker)" + self.node.get("container_name")
return self.host_ip
Loading

0 comments on commit 40683e5

Please sign in to comment.