From 7471897165e2f5dfb3d882f9f4a91d218f6ad8c5 Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 09:16:54 -0500 Subject: [PATCH 01/16] black --- src/cloudmesh/common/Benchmark.py | 26 +- src/cloudmesh/common/DateTime.py | 11 +- src/cloudmesh/common/FlatDict.py | 38 +-- src/cloudmesh/common/Host.py | 327 ++++++++++++--------- src/cloudmesh/common/JobMultiHostScript.py | 17 +- src/cloudmesh/common/JobScript.py | 58 ++-- src/cloudmesh/common/JobSet.py | 105 ++++--- src/cloudmesh/common/Printer.py | 288 +++++++++--------- src/cloudmesh/common/StopWatch.py | 284 +++++++++--------- src/cloudmesh/common/Tabulate.py | 122 ++++---- src/cloudmesh/common/console.py | 114 +++---- src/cloudmesh/common/debug.py | 48 +-- src/cloudmesh/common/default.py | 14 +- src/cloudmesh/common/deprecated.py | 17 +- src/cloudmesh/common/location.py | 1 + src/cloudmesh/common/logger.py | 12 +- src/cloudmesh/common/parameter.py | 26 +- src/cloudmesh/common/prettytable.py | 242 +++++++++------ src/cloudmesh/common/security.py | 6 +- src/cloudmesh/common/strdb.py | 14 +- src/cloudmesh/common/sudo.py | 6 +- src/cloudmesh/common/systeminfo.py | 85 +++--- src/cloudmesh/common/util.py | 160 +++++----- src/cloudmesh/common/variables.py | 65 ++-- src/cloudmesh/common/wifi.py | 21 +- 25 files changed, 1159 insertions(+), 948 deletions(-) diff --git a/src/cloudmesh/common/Benchmark.py b/src/cloudmesh/common/Benchmark.py index 6887768..9ac4fb0 100644 --- a/src/cloudmesh/common/Benchmark.py +++ b/src/cloudmesh/common/Benchmark.py @@ -6,8 +6,7 @@ from cloudmesh.common.StopWatch import StopWatch from cloudmesh.common.util import path_expand from cloudmesh.common.variables import Variables -# -# + # pylint: disable=C0103 # noinspection PyPep8Naming @@ -31,7 +30,6 @@ class Benchmark(object): This class relies on the StopWatch class for timer functionality. """ - @staticmethod def debug(): """Sets the CMS shell variables for trace, debug, and verbosity. @@ -107,11 +105,13 @@ def Stop(): StopWatch.status(Benchmark.name(with_class=True), True) @staticmethod - def print(sysinfo=True, - csv=True, - tag=None, - node=None, - user=None, ): + def print( + sysinfo=True, + csv=True, + tag=None, + node=None, + user=None, + ): """prints the benchmark information with all timers Args: @@ -142,17 +142,13 @@ def yaml(path, n): Usage: Benchmark.yaml("./example.yaml", 10) """ - cm = { - "cloudmesh": {} - } + cm = {"cloudmesh": {}} for i in range(0, n): - cm["cloudmesh"][f"service{i}"] = { - "attribute": f"service{i}" - } + cm["cloudmesh"][f"service{i}"] = {"attribute": f"service{i}"} pprint(cm) location = path_expand(path) - with open(location, 'w') as yaml_file: + with open(location, "w") as yaml_file: yaml.dump(cm, yaml_file, default_flow_style=False) # noinspection SpellCheckingInspection diff --git a/src/cloudmesh/common/DateTime.py b/src/cloudmesh/common/DateTime.py index fa21d38..60baf11 100644 --- a/src/cloudmesh/common/DateTime.py +++ b/src/cloudmesh/common/DateTime.py @@ -36,8 +36,10 @@ class DateTime(object): @staticmethod def now(): - return TIME.datetime.now(TIME.timezone) - + # return TIME.datetime.now(TIME.timezone) + # return TIME.datetime.now(TIME.timezone.utc) + return TIME.datetime.now() + @staticmethod def local_now(): return TIME.datetime.now() @@ -85,10 +87,11 @@ def local(time): return DateTime.utc_to_local(time) def utc_to_local(time): - TIME_FORMAT = '%Y-%m-%d %H:%M:%S' + TIME_FORMAT = "%Y-%m-%d %H:%M:%S" utc = TIME.datetime.utcnow().strftime(TIME_FORMAT) timestamp = calendar.timegm( - (TIME.datetime.strptime(utc, TIME_FORMAT)).timetuple()) + (TIME.datetime.strptime(utc, TIME_FORMAT)).timetuple() + ) local = TIME.datetime.fromtimestamp(timestamp).strftime(TIME_FORMAT) return local + " " + str(DateTime.timezone) diff --git a/src/cloudmesh/common/FlatDict.py b/src/cloudmesh/common/FlatDict.py index 82ef37d..977d81b 100644 --- a/src/cloudmesh/common/FlatDict.py +++ b/src/cloudmesh/common/FlatDict.py @@ -68,7 +68,7 @@ def flatme(d): return o -def flatten(d, parent_key='', sep='__'): +def flatten(d, parent_key="", sep="__"): """flattens the dict into a one dimensional dictionary Args: @@ -299,17 +299,19 @@ def load(self, content=None, expand=True, sep="."): elif type(content) == dict: self.loadd(content=content, sep=".") elif os.path.isfile(str(content)): - print ("file") + print("file") self.loadf(filename=content, sep=".") elif type(content) == str: self.loads(content=content, sep=".") else: config = None self.__init__(config, sep=sep) - e = expand_config_parameters(flat=self.__dict__, - expand_yaml=True, - expand_os=self.expand_os, - expand_cloudmesh=self.expand_cloudmesh or self.expand_cm) + e = expand_config_parameters( + flat=self.__dict__, + expand_yaml=True, + expand_os=self.expand_os, + expand_cloudmesh=self.expand_cloudmesh or self.expand_cm, + ) self.__dict__ = e def apply_in_string(self, content): @@ -398,8 +400,7 @@ def object_to_dict(cls, obj): return dict_obj -def read_config_parameters(filename=None, - d=None): +def read_config_parameters(filename=None, d=None): """This file reads in configuration parameters defined in a yaml file and produces a flattend dict. It reads in the yaml date from a filename and/or a string. If both are specified the data in the filename will be read first @@ -538,12 +539,14 @@ def read_config_parameters_from_dict(content=None, d=None): return config -def expand_config_parameters(flat=None, - expand_yaml=True, - expand_os=True, - expand_cloudmesh=True, - debug=False, - depth=100): +def expand_config_parameters( + flat=None, + expand_yaml=True, + expand_os=True, + expand_cloudmesh=True, + debug=False, + depth=100, +): """expands all variables in the flat dict if they are specified in the values of the flatdict. Args: @@ -588,7 +591,7 @@ def expand_config_parameters(flat=None, if expand_yaml: found = True - for i in range (0,depth): + for i in range(0, depth): for variable in flat.keys(): name = "{" + variable + "}" value = flat[variable] @@ -597,7 +600,6 @@ def expand_config_parameters(flat=None, print("found", variable, "->", value) txt = txt.replace(name, str(value)) - if "{os." in values and expand_os: for variable in os.environ: if variable != "_": @@ -648,7 +650,7 @@ def expand_config_parameters(flat=None, return config -''' +""" def main(): @@ -736,4 +738,4 @@ def main(): if __name__ == "__main__": main() -''' +""" diff --git a/src/cloudmesh/common/Host.py b/src/cloudmesh/common/Host.py index ee7b4f1..391738c 100644 --- a/src/cloudmesh/common/Host.py +++ b/src/cloudmesh/common/Host.py @@ -14,13 +14,15 @@ class Host(object): - - def _print(results, output='table'): - - if output in ['table', 'yaml']: - print(Printer.write(results, - order=['host', 'success', 'stdout', 'stderr'], - output=output)) + def _print(results, output="table"): + if output in ["table", "yaml"]: + print( + Printer.write( + results, + order=["host", "success", "stdout", "stderr"], + output=output, + ) + ) else: pprint(results) @@ -49,11 +51,25 @@ def get_hostnames(names): return manager, workers @staticmethod - def config(hosts=None, - ips=None, - username=None, - key="~/.ssh/id_rsa"): + def config(hosts=None, ips=None, username=None, key="~/.ssh/id_rsa"): + """ + Configures the SSH connection parameters for a set of hosts. + + This method expands the hosts and IPs if they are not lists, checks if the number of hosts and IPs match, + and then creates the SSH configuration for each host. + Parameters: + hosts (str or list): A single host or a list of hosts to configure. If a string is provided, it is expanded into a list. + ips (str or list): A single IP or a list of IPs corresponding to the hosts. If a string is provided, it is expanded into a list. + username (str): The username to use for the SSH connections. + key (str): The path to the SSH key to use for the connections. Defaults to "~/.ssh/id_rsa". + + Returns: + str: The SSH configuration for the hosts. + + Raises: + ValueError: If the number of hosts and IPs provided do not match. + """ if type(hosts) != list: _hosts = Parameter.expand(hosts) if ips is not None: @@ -64,7 +80,6 @@ def config(hosts=None, result = "" for i in range(0, len(_hosts)): - host = _hosts[i] hostname = "" @@ -75,17 +90,19 @@ def config(hosts=None, ip = _ips[i] hostname = f"Hostname {ip}" - data = textwrap.dedent(f""" + data = textwrap.dedent( + f""" Host {host} StrictHostKeyChecking no LogLevel ERROR UserKnownHostsFile /dev/null IdentityFile {key} {hostname} - """) + """ + ) if username: - data += (f" {user}\n") + data += f" {user}\n" result += data return result @@ -129,13 +146,10 @@ def _run(args): else: command = args.get("command") - result = subprocess.run( - command, - capture_output=True, - shell=shell) + result = subprocess.run(command, capture_output=True, shell=shell) result.stdout = result.stdout.decode("utf-8", "ignore").strip() - if result.stderr == b'': + if result.stderr == b"": result.stderr = None stderr = result.stderr @@ -143,15 +157,15 @@ def _run(args): stdout = result.stdout data = { - 'host': args.get("host"), - 'command': args.get("command"), - 'execute': args.get("execute"), - 'stdout': stdout, - 'stderr': stderr, - 'returncode': returncode, - 'success': returncode == 0, - 'date': DateTime.now(), - 'cmd': " ".join(args.get("command")) + "host": args.get("host"), + "command": args.get("command"), + "execute": args.get("execute"), + "stdout": stdout, + "stderr": stderr, + "returncode": returncode, + "success": returncode == 0, + "date": DateTime.now(), + "cmd": " ".join(args.get("command")), } except Exception as e: print(e) @@ -159,12 +173,9 @@ def _run(args): return data @staticmethod - def run(hosts=None, - command=None, - execute=None, - processors=3, - shell=False, - **kwargs): + def run( + hosts=None, command=None, execute=None, processors=3, shell=False, **kwargs + ): """Executes the command on all hosts. The key values specified in **kwargs will be replaced prior to the execution. Furthermore, {host} will be replaced with the @@ -186,11 +197,15 @@ def run(hosts=None, """ hosts = Parameter.expand(hosts) - args = [{'command': [c.format(host=host, **kwargs) for c in command], - 'shell': shell, - 'host': host, - 'execute': execute, - } for host in hosts] + args = [ + { + "command": [c.format(host=host, **kwargs) for c in command], + "shell": shell, + "host": host, + "execute": execute, + } + for host in hosts + ] if "executor" not in args: _executor = Host._run @@ -204,15 +219,17 @@ def run(hosts=None, return res @staticmethod - def ssh(hosts=None, - command=None, - username=None, - key="~/.ssh/id_rsa", - processors=3, - dryrun=False, # notused - executor=None, - verbose=False, # not used - **kwargs): + def ssh( + hosts=None, + command=None, + username=None, + key="~/.ssh/id_rsa", + processors=3, + dryrun=False, # notused + executor=None, + verbose=False, # not used + **kwargs, + ): # # BUG: this code has a bug and does not deal with different # usernames on the host to be checked. @@ -233,32 +250,42 @@ def ssh(hosts=None, key = path_expand(key) - ssh_command = ['ssh', - '-o', 'StrictHostKeyChecking=no', - '-o', 'UserKnownHostsFile=/dev/null', - '-o', 'PreferredAuthentications=publickey', - '-i', key, - '{host}', - f'{command}'] - result = Host.run(hosts=hosts, - command=ssh_command, - execute=command, - shell=False, - executor=executor, - **kwargs) + ssh_command = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "PreferredAuthentications=publickey", + "-i", + key, + "{host}", + f"{command}", + ] + result = Host.run( + hosts=hosts, + command=ssh_command, + execute=command, + shell=False, + executor=executor, + **kwargs, + ) return result @staticmethod - def put(hosts=None, - source=None, - destination=None, - username=None, - key="~/.ssh/id_rsa", - shell=False, - processors=3, - dryrun=False, - verbose=False): + def put( + hosts=None, + source=None, + destination=None, + username=None, + key="~/.ssh/id_rsa", + shell=False, + processors=3, + dryrun=False, + verbose=False, + ): """ Args: command: the command to be executed @@ -275,28 +302,32 @@ def put(hosts=None, key = path_expand(key) - command = ['scp', - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - '-i', key, - source, - "{host}:{destination}"] + command = [ + "scp", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-i", + key, + source, + "{host}:{destination}", + ] execute = f"cp {source} {destination}" - result = Host.run(hosts=hosts, - command=command, - execute=execute, - destination=destination, - shell=False) + result = Host.run( + hosts=hosts, + command=command, + execute=execute, + destination=destination, + shell=False, + ) return result @staticmethod - def check(hosts=None, - username=None, - key="~/.ssh/id_rsa", - processors=3): + def check(hosts=None, username=None, key="~/.ssh/id_rsa", processors=3): # # BUG: this code has a bug and does not deal with different # usernames on the host to be checked. @@ -313,11 +344,13 @@ def check(hosts=None, """ hosts = Parameter.expand(hosts) - result = Host.ssh(hosts=hosts, - command='hostname', - username=username, - key=key, - processors=processors) + result = Host.ssh( + hosts=hosts, + command="hostname", + username=username, + key=key, + processors=processors, + ) return result @@ -333,25 +366,27 @@ def _ping(args): a dict representing the result, if returncode=0 ping is successfully """ - ip = args['ip'] - count = str(args['count']) - - count_flag = '-n' if os_is_windows() else '-c' + ip = args["ip"] + count = str(args["count"]) + + count_flag = "-n" if os_is_windows() else "-c" if os_is_windows(): # adding ipv4 enforce for windows # for some reason -4 is required or hosts # fail. we need ipv4 - command = ['ping', '-4', ip, count_flag, count] + command = ["ping", "-4", ip, count_flag, count] else: - command = ['ping', count_flag, count, ip] + command = ["ping", count_flag, count, ip] # command = ['ping', '-4', ip, count_flag, count] result = subprocess.run(command, capture_output=True) try: - timers = result.stdout \ - .decode("utf-8", "ignore") \ - .split("round-trip min/avg/max/stddev =")[1] \ - .replace('ms', '').strip() \ + timers = ( + result.stdout.decode("utf-8", "ignore") + .split("round-trip min/avg/max/stddev =")[1] + .replace("ms", "") + .strip() .split("/") + ) data = { "host": ip, "success": result.returncode == 0, @@ -359,7 +394,7 @@ def _ping(args): "min": timers[0], "avg": timers[1], "max": timers[2], - "stddev": timers[3] + "stddev": timers[3], } except: # noqa: E722 data = { @@ -386,7 +421,7 @@ def ping(hosts=None, count=1, processors=3): hosts = Parameter.expand(hosts) # wrap ip and count into one list to be sent to Pool map - args = [{'ip': ip, 'count': count} for ip in hosts] + args = [{"ip": ip, "count": count} for ip in hosts] with Pool(processors) as p: res = p.map(Host._ping, args) @@ -396,12 +431,14 @@ def ping(hosts=None, count=1, processors=3): return res @staticmethod - def ssh_keygen(hosts=None, - filename="~/.ssh/id_rsa", - username=None, - processors=3, - dryrun=False, - verbose=True): + def ssh_keygen( + hosts=None, + filename="~/.ssh/id_rsa", + username=None, + processors=3, + dryrun=False, + verbose=True, + ): """generates the keys on the specified hosts. this fonction does not work well as it still will aski if we overwrite. @@ -418,26 +455,32 @@ def ssh_keygen(hosts=None, """ hosts = Parameter.expand(hosts) command = f'ssh-keygen -q -N "" -f {filename} <<< y' - result_keys = Host.ssh(hosts=hosts, - command=command, - username=username, - dryrun=dryrun, - processors=processors, - executor=os.system) + result_keys = Host.ssh( + hosts=hosts, + command=command, + username=username, + dryrun=dryrun, + processors=processors, + executor=os.system, + ) Host._print(result_keys) - result_keys = Host.ssh(hosts=hosts, - processors=processors, - command='cat .ssh/id_rsa.pub', - username=username) + result_keys = Host.ssh( + hosts=hosts, + processors=processors, + command="cat .ssh/id_rsa.pub", + username=username, + ) return result_keys @staticmethod - def gather_keys(username=None, - hosts=None, - filename="~/.ssh/id_rsa.pub", - key="~/.ssh/id_rsa", - processors=3, - dryrun=False): + def gather_keys( + username=None, + hosts=None, + filename="~/.ssh/id_rsa.pub", + key="~/.ssh/id_rsa", + processors=3, + dryrun=False, + ): """returns in a list the keys of the specified hosts Args: @@ -452,10 +495,12 @@ def gather_keys(username=None, """ names = Parameter.expand(hosts) - results_key = Host.ssh(hosts=names, - command='cat ~/.ssh/id_rsa.pub', - username=username, - verbose=False) + results_key = Host.ssh( + hosts=names, + command="cat ~/.ssh/id_rsa.pub", + username=username, + verbose=False, + ) # results_authorized = Host.ssh(hosts=names, # command='cat .ssh/id_rsa.pub', # username=username, @@ -464,28 +509,28 @@ def gather_keys(username=None, filename = path_expand(filename) content = readfile(filename).strip() localkey = { - 'host': "localhost", - 'command': [''], - 'execute': "", - 'stdout': content, - 'stderr': None, - 'returncode': True, - 'success': True, - 'date': DateTime.now() + "host": "localhost", + "command": [""], + "execute": "", + "stdout": content, + "stderr": None, + "returncode": True, + "success": True, + "date": DateTime.now(), } if results_key is None: # and results_authorized is None: return "" # getting the output and also removing duplicates - keys = [localkey['stdout'].strip()] + keys = [localkey["stdout"].strip()] for element in results_key: - if element['stdout'] != '': - keys.append(element['stdout'].strip()) + if element["stdout"] != "": + keys.append(element["stdout"].strip()) output = list(set(keys)) - output = '\n'.join(output) + output = "\n".join(output) print(output) return output diff --git a/src/cloudmesh/common/JobMultiHostScript.py b/src/cloudmesh/common/JobMultiHostScript.py index 5037de5..70e4d72 100755 --- a/src/cloudmesh/common/JobMultiHostScript.py +++ b/src/cloudmesh/common/JobMultiHostScript.py @@ -63,8 +63,14 @@ def run(self, beginLine=None, endLine=None): job.Print() @staticmethod - def execute(script, name="script", hosts=None, executor=JobSet.ssh, - beginLine=None, endLine=None): + def execute( + script, + name="script", + hosts=None, + executor=JobSet.ssh, + beginLine=None, + endLine=None, + ): job = JobMultiHostScript(name, script, hosts, executor) job.run(beginLine, endLine) @@ -107,7 +113,7 @@ def cms(self, arguments): return -if __name__ == '__main__': +if __name__ == "__main__": script = """ # This is a comment @@ -119,5 +125,6 @@ def cms(self, arguments): """ hosts = Parameter.expand("purple[01-02]") - result = JobMultiHostScript.execute(script, "script_name", hosts, - beginLine="# Task: pwd", endLine="# Task: uname") + result = JobMultiHostScript.execute( + script, "script_name", hosts, beginLine="# Task: pwd", endLine="# Task: uname" + ) diff --git a/src/cloudmesh/common/JobScript.py b/src/cloudmesh/common/JobScript.py index a57b57e..e3f320a 100644 --- a/src/cloudmesh/common/JobScript.py +++ b/src/cloudmesh/common/JobScript.py @@ -64,8 +64,7 @@ class JobScript: def __init__(self): self.script = None - def run(self, script=None, name="script", host=None, executor=JobSet.ssh, - **kwargs): + def run(self, script=None, name="script", host=None, executor=JobSet.ssh, **kwargs): # Prepare parameters if script is None: Console.error("The script is not defined, found None as content") @@ -94,23 +93,24 @@ def run(self, script=None, name="script", host=None, executor=JobSet.ssh, line, tag = line.split("# tag:", 1) tag = tag.strip() line = line.strip() - jobs.add_directory({ - "script": name, - "name": tag, - "tag": tag, - "line": counter, - "host": host, - "counter": counter, - "command": line - }) + jobs.add_directory( + { + "script": name, + "name": tag, + "tag": tag, + "line": counter, + "host": host, + "counter": counter, + "command": line, + } + ) counter = counter + 1 jobs.run(parallel=1) self.job = jobs.job return self.job @staticmethod - def execute(script, name="script", host=None, executor=JobSet.ssh, - **kwargs): + def execute(script, name="script", host=None, executor=JobSet.ssh, **kwargs): job = JobScript() job.run(script=script, name=name, host=host, **kwargs) return job.array() @@ -123,32 +123,40 @@ def array(self): return [self.job[x] for x in self.job] -if __name__ == '__main__': +if __name__ == "__main__": # # Tow ways to run. # # Static method - result = JobScript.execute(""" + result = JobScript.execute( + """ # This is a comment pwd # tag: pwd uname -a - """) - print(Printer.write( - result, - order=["name", "command", "status", "stdout", "returncode"])) + """ + ) + print( + Printer.write( + result, order=["name", "command", "status", "stdout", "returncode"] + ) + ) # Variables job = JobScript() - job.run(name="variable script", - script=""" + job.run( + name="variable script", + script=""" # This is a comment pwd # tag: pwd uname -a - """) - print(Printer.write( - job.array(), - order=["name", "command", "status", "stdout", "returncode"])) + """, + ) + print( + Printer.write( + job.array(), order=["name", "command", "status", "stdout", "returncode"] + ) + ) # # Partial example output diff --git a/src/cloudmesh/common/JobSet.py b/src/cloudmesh/common/JobSet.py index d25dca6..5dd2f0c 100644 --- a/src/cloudmesh/common/JobSet.py +++ b/src/cloudmesh/common/JobSet.py @@ -94,20 +94,20 @@ def ssh(spec): hostname = platform.uname()[1] local = hostname == spec.host - if 'key' not in spec: + if "key" not in spec: spec.key = path_expand("~/.ssh/id_rsa.pub") - ssh = \ - f'ssh' \ - f' -o StrictHostKeyChecking=no' \ - f' -o UserKnownHostsFile=/dev/null' \ - f' -i {spec.key} {spec.host}' + ssh = ( + f"ssh" + f" -o StrictHostKeyChecking=no" + f" -o UserKnownHostsFile=/dev/null" + f" -i {spec.key} {spec.host}" + ) # result.stdout = result.stdout.decode("utf-8", "ignore") if "os" in spec: - - if 'tmp' not in spec: + if "tmp" not in spec: spec.tmp = "/tmp" tee = f"tee {spec.tmp}/cloudmesh.{spec.name}" if local: @@ -122,13 +122,15 @@ def ssh(spec): if local: command = f"{spec.command} " else: - command = f"{ssh} \'{spec.command}\'" + command = f"{ssh} '{spec.command}'" # print ("RUN check_output", command) try: stderr = "" returncode = 0 - result = subprocess.check_output(command, shell=True, stderr=subprocess.PIPE) + result = subprocess.check_output( + command, shell=True, stderr=subprocess.PIPE + ) except subprocess.CalledProcessError as grepexc: # print("Error Code", grepexc.returncode, grepexc.output) result = "Command could not run | Error Code: ", grepexc.returncode @@ -136,43 +138,47 @@ def ssh(spec): stderr = grepexc.stderr returncode = grepexc.returncode - return dict({ - "name": spec.name, - "stdout": result, - "stderr": stderr, - "returncode": returncode, - "status": "defined" - }) + return dict( + { + "name": spec.name, + "stdout": result, + "stderr": stderr, + "returncode": returncode, + "status": "defined", + } + ) @staticmethod def execute(spec): result = subprocess.check_output(spec["command"], shell=True) - return dict({ - "name": spec["name"], - "stdout": result, - "stderr": "", - "returncode": 0, - "status": "defined" - }) + return dict( + { + "name": spec["name"], + "stdout": result, + "stderr": "", + "returncode": 0, + "status": "defined", + } + ) @staticmethod def identity(entry_with_name): - - return dict({ - "name": entry_with_name["name"], - "stdout": "identity " + entry_with_name["name"], - "stderr": "", - "returncode": 0, - "status": "defined" - }) + return dict( + { + "name": entry_with_name["name"], + "stdout": "identity " + entry_with_name["name"], + "stderr": "", + "returncode": 0, + "status": "defined", + } + ) def add(self, spec, executor=None): name = spec["name"] self.job[name] = spec self.job[name]["status"] = "defined" - self.job[name]["executor"] = spec.get( - "executor") or executor or self.executor + self.job[name]["executor"] = spec.get("executor") or executor or self.executor def _run(self, spec): result = dict(spec) @@ -184,7 +190,6 @@ def _run(self, spec): return result def run(self, parallel=3): - if len(self.job) == 0: res = None elif len(self.job) == 1: @@ -201,7 +206,7 @@ def run(self, parallel=3): p.join() for entry in res: - name = entry['name'] + name = entry["name"] for a in entry: self.job[name].update(entry) @@ -227,16 +232,18 @@ def array(self): return [self.job[x] for x in self.job] -if __name__ == '__main__': - def command_execute(spec): - return dict({ - "name": spec["name"], - "stdout": "command " + spec["command"], - "stderr": "", - "returncode": 0, - "status": "defined" - }) +if __name__ == "__main__": + def command_execute(spec): + return dict( + { + "name": spec["name"], + "stdout": "command " + spec["command"], + "stderr": "", + "returncode": 0, + "status": "defined", + } + ) hostname = platform.uname()[1] @@ -296,6 +303,8 @@ def command_execute(spec): t.add({"name": hostname, "host": hostname, "command": "uname -a"}) t.run(parallel=3) - print(Printer.write(t.array(), - order=["name", "command", "status", "stdout", - "returncode"])) + print( + Printer.write( + t.array(), order=["name", "command", "status", "stdout", "returncode"] + ) + ) diff --git a/src/cloudmesh/common/Printer.py b/src/cloudmesh/common/Printer.py index 011dce8..6648119 100644 --- a/src/cloudmesh/common/Printer.py +++ b/src/cloudmesh/common/Printer.py @@ -6,6 +6,7 @@ from cloudmesh.common.FlatDict import flatten from cloudmesh.common.console import Console from cloudmesh.common.dotdict import dotdict + # from prettytable import PrettyTable from cloudmesh.common.prettytable import PrettyTable from cloudmesh.common.util import convert_from_unicode @@ -16,16 +17,18 @@ class Printer(object): """A simple Printer class with convenient methods to print dictionary, tables, csv, lists""" @classmethod - def flatwrite(cls, table, - order=None, - header=None, - output="table", - sort_keys=True, - show_none="", - humanize=None, - sep=".", - max_width=48 - ): + def flatwrite( + cls, + table, + order=None, + header=None, + output="table", + sort_keys=True, + show_none="", + humanize=None, + sep=".", + max_width=48, + ): """Writes the information given in the table. Args: @@ -45,25 +48,28 @@ def flatwrite(cls, table, """ flat = flatten(table, sep=sep) - return Printer.write(flat, - sort_keys=sort_keys, - order=order, - header=header, - output=output, - humanize=humanize, - max_width=max_width - ) + return Printer.write( + flat, + sort_keys=sort_keys, + order=order, + header=header, + output=output, + humanize=humanize, + max_width=max_width, + ) @classmethod - def write(cls, table, - order=None, - header=None, - output="table", - sort_keys=True, - humanize=None, - show_none="", - max_width=48 - ): + def write( + cls, + table, + order=None, + header=None, + output="table", + sort_keys=True, + humanize=None, + show_none="", + max_width=48, + ): """writes the information given in the table Args: @@ -89,42 +95,46 @@ def write(cls, table, elif table is None: return None elif type(table) in [dict, dotdict]: - return cls.dict(table, - order=order, - header=header, - output=output, - sort_keys=sort_keys, - humanize=humanize, - show_none=show_none, - max_width=max_width) + return cls.dict( + table, + order=order, + header=header, + output=output, + sort_keys=sort_keys, + humanize=humanize, + show_none=show_none, + max_width=max_width, + ) elif type(table) == list: - - return cls.list(table, - order=order, - header=header, - output=output, - sort_keys=sort_keys, - humanize=humanize, - show_none=show_none, - max_width=max_width) + return cls.list( + table, + order=order, + header=header, + output=output, + sort_keys=sort_keys, + humanize=humanize, + show_none=show_none, + max_width=max_width, + ) else: Console.error("unknown type {0}".format(type(table))) @classmethod - def list(cls, - l, # noqa: E741 - order=None, - header=None, - output="table", - sort_keys=True, - humanize=None, - show_none="", - max_width=48 - ): + def list( + cls, + l, # noqa: E741 + order=None, + header=None, + output="table", + sort_keys=True, + humanize=None, + show_none="", + max_width=48, + ): """ This method takes a list as input and formats it for printing in a tabular format. - + Args: l (list): The input list to be formatted. order (list): The order in which the columns should be displayed. @@ -134,35 +144,40 @@ def list(cls, humanize (bool): Whether to humanize the values. Default is None. show_none (str): The string to display for None values. Default is "". max_width (int): The maximum width for a cell. Default is 48. - + Returns: dict: A dictionary containing the formatted list. """ - + d = {} count = 0 for entry in l: name = str(count) d[name] = entry count += 1 - return cls.dict(d, - order=order, - header=header, - sort_keys=sort_keys, - output=output, - humanize=humanize, - show_none=show_none, - max_width=max_width) + return cls.dict( + d, + order=order, + header=header, + sort_keys=sort_keys, + output=output, + humanize=humanize, + show_none=show_none, + max_width=max_width, + ) @classmethod - def dict(cls, d, - order=None, - header=None, - output="table", - sort_keys=True, - humanize=None, - show_none="", - max_width=48): + def dict( + cls, + d, + order=None, + header=None, + output="table", + sort_keys=True, + humanize=None, + show_none="", + max_width=48, + ): """ Prints a dictionary in various output formats. @@ -180,34 +195,36 @@ def dict(cls, d, Returns: str: The formatted dictionary based on the specified output format. - """ + """ if output in ["table", "filter"]: if d == {}: return None else: - return cls.dict_table(d, - order=order, - header=header, - humanize=humanize, - sort_keys=sort_keys, - max_width=max_width) + return cls.dict_table( + d, + order=order, + header=header, + humanize=humanize, + sort_keys=sort_keys, + max_width=max_width, + ) elif output == "html": # does not work if d == []: return "Empty data" else: - return cls.dict_html(d, - order=order, - header=header, - humanize=humanize, - sort_keys=sort_keys, - max_width=max_width) + return cls.dict_html( + d, + order=order, + header=header, + humanize=humanize, + sort_keys=sort_keys, + max_width=max_width, + ) elif output == "csv": - return cls.csv(d, - order=order, - header=header, - humanize=humanize, - sort_keys=sort_keys) + return cls.csv( + d, order=order, header=header, humanize=humanize, sort_keys=sort_keys + ) elif output == "json": return json.dumps(d, sort_keys=sort_keys, indent=4) elif output == "yaml": @@ -218,8 +235,7 @@ def dict(cls, d, return "UNKNOWN FORMAT. Please use table, csv, json, yaml, dict." @classmethod - def csv(cls, d, order=None, header=None, humanize=None, - sort_keys=True): + def csv(cls, d, order=None, header=None, humanize=None, sort_keys=True): """prints a table in csv format Args: @@ -244,7 +260,7 @@ def _get(element, key): try: tmp = str(d[element][key]) except: # noqa: E722 - tmp = ' ' + tmp = " " return tmp if d is None or d == {}: @@ -281,31 +297,39 @@ def _get(element, key): return table @classmethod - def dict_html(cls, d, - order=None, - header=None, - sort_keys=True, - show_none="", - humanize=None, - max_width=48): - x = Printer.dict_table(d, - order=order, - header=header, - sort_keys=sort_keys, - show_none=show_none, - humanize=humanize, - max_width=max_width) + def dict_html( + cls, + d, + order=None, + header=None, + sort_keys=True, + show_none="", + humanize=None, + max_width=48, + ): + x = Printer.dict_table( + d, + order=order, + header=header, + sort_keys=sort_keys, + show_none=show_none, + humanize=humanize, + max_width=max_width, + ) return x.get_html_string() @classmethod - def dict_table(cls, d, - order=None, - header=None, - sort_keys=True, - show_none="", - humanize=None, - max_width=48): + def dict_table( + cls, + d, + order=None, + header=None, + sort_keys=True, + show_none="", + humanize=None, + max_width=48, + ): """prints a pretty table from an dict of dicts Args: @@ -341,7 +365,7 @@ def _get(item, key): tmp = DateTime.humanize(start - tmp) # tmp = naturaltime(start - tmp) except: # noqa: E722 - tmp = ' ' + tmp = " " return tmp if d is None or d == {}: @@ -362,8 +386,9 @@ def _get(item, key): if type(sort_keys) is str: sorted_list = sorted(d, key=lambda x: d[x][sort_keys]) elif type(sort_keys) == tuple: - sorted_list = sorted(d, key=lambda x: tuple( - [d[x][sort_key] for sort_key in sort_keys])) + sorted_list = sorted( + d, key=lambda x: tuple([d[x][sort_key] for sort_key in sort_keys]) + ) else: sorted_list = d else: @@ -379,9 +404,9 @@ def _get(item, key): return x @classmethod - def attribute(cls, d, header=None, order=None, sort_keys=True, - humanize=None, - output="table"): + def attribute( + cls, d, header=None, order=None, sort_keys=True, humanize=None, output="table" + ): """prints a attribute/key value table Args: @@ -427,7 +452,7 @@ def attribute(cls, d, header=None, order=None, sort_keys=True, return cls.dict({output: d}, output=output) @classmethod - def print_list(cls, l, output='table'): # noqa: E741 + def print_list(cls, l, output="table"): # noqa: E741 """prints a list Args: @@ -438,7 +463,7 @@ def print_list(cls, l, output='table'): # noqa: E741 """ - def dict_from_list(l): # noqa: E741 + def dict_from_list(l): # noqa: E741 """returns a dict from a list for printing Args: @@ -450,27 +475,27 @@ def dict_from_list(l): # noqa: E741 d = dict([(idx, item) for idx, item in enumerate(l)]) return d - if output == 'table': + if output == "table": x = PrettyTable(["Index", "Host"]) - for (idx, item) in enumerate(l): + for idx, item in enumerate(l): x.add_row([idx, item]) x.align = "l" x.align["Index"] = "r" return x - elif output == 'csv': + elif output == "csv": return ",".join(l) - elif output == 'dict': + elif output == "dict": d = dict_from_list(l) return d - elif output == 'json': + elif output == "json": d = dict_from_list(l) result = json.dumps(d, indent=4) return result - elif output == 'yaml': + elif output == "yaml": d = dict_from_list(l) result = yaml.dump(d, default_flow_style=False) return result - elif output == 'txt': + elif output == "txt": return "\n".join(l) @classmethod @@ -498,8 +523,7 @@ def row_table(cls, d, order=None, labels=None): value_keys = list(value) first_key = value_keys[0] rest_keys = value_keys[1:] - x.add_row( - [key, "{0} : {1}".format(first_key, value[first_key])]) + x.add_row([key, "{0} : {1}".format(first_key, value[first_key])]) for element in rest_keys: x.add_row(["", "{0} : {1}".format(element, value[element])]) else: diff --git a/src/cloudmesh/common/StopWatch.py b/src/cloudmesh/common/StopWatch.py index 9f92257..ab8922b 100644 --- a/src/cloudmesh/common/StopWatch.py +++ b/src/cloudmesh/common/StopWatch.py @@ -101,17 +101,20 @@ from cloudmesh.common.util import readfile from cloudmesh.common.util import writefile -def progress(filename=None, # + - status="ready", # + - progress: Union[int, str, float] = 0, # + - pid=None, # + - time=False, - stdout=True, - stderr=True, - append=None, - with_banner=False, - # variable we do not have, but should be in kwrags - **kwargs): + +def progress( + filename=None, # + + status="ready", # + + progress: Union[int, str, float] = 0, # + + pid=None, # + + time=False, + stdout=True, + stderr=True, + append=None, + with_banner=False, + # variable we do not have, but should be in kwrags + **kwargs, +): """Creates a printed line of the form "# cloudmesh status=ready progress=0 pid=$$ time='2022-08-05 16:29:40.228901'" @@ -135,7 +138,7 @@ def progress(filename=None, # + Returns: str: progress string """ - if type(progress) in ['int', 'float']: + if type(progress) in ["int", "float"]: progress = str(progress) if pid is None: if "SLURM_JOB_ID" in os.environ: @@ -148,7 +151,7 @@ def progress(filename=None, # + msg = f"# cloudmesh status={status} progress={progress} pid={pid}\n" if time: t = str(DateTime.now()) - msg = msg + f" time='{t}'" + msg = msg + f" time='{t}'" if kwargs: for name, value in kwargs.items(): variables = variables + f" {name}={value}" @@ -165,6 +168,7 @@ def progress(filename=None, # + appendfile(filename, msg) return msg + def rename(newname): """decorator to rename a function @@ -204,6 +208,7 @@ def wrapper(*args, **kwargs): class StopWatch(object): """A class to measure times between events.""" + debug = False verbose = True # Timer start dict @@ -269,7 +274,6 @@ class StopWatch(object): # } # config["benchmark"].update(argv) - @classmethod def keys(cls): """returns the names of the timers""" @@ -333,7 +337,6 @@ def event(cls, name, msg=None, values=None, value=None, stack_offset=2): if cls.debug: print("Timer", name, "event ...") - @classmethod def start(cls, name, values=None, value=None): """starts a timer with the given name. @@ -348,7 +351,6 @@ def start(cls, name, values=None, value=None): """ values = values or value - if cls.debug: print("Timer", name, "started ...") if name not in cls.timer_sum: @@ -363,7 +365,6 @@ def start(cls, name, values=None, value=None): if cls.debug: print("Timer", name, "start ...") - @classmethod def stop(cls, name, state=True, values=None, value=None): """stops the timer with a given name. @@ -380,7 +381,9 @@ def stop(cls, name, state=True, values=None, value=None): cls.timer_end[name] = time.time() # if cumulate: # cls.timer_end[name] = cls.timer_end[name] + cls.timer_last[name] - cls.timer_sum[name] = cls.timer_sum[name] + cls.timer_end[name] - cls.timer_start[name] + cls.timer_sum[name] = ( + cls.timer_sum[name] + cls.timer_end[name] - cls.timer_start[name] + ) cls.timer_status[name] = state if values: StopWatch.timer_values[name] = values @@ -490,14 +493,15 @@ def __str__(cls): """ s = "" for t in cls.timer_end: - data = {"label": t, - "start": str(cls.timer_start[t]), - "end": str(cls.timer_end[t]), - "status": str(cls.timer_status[t]), - "elapsed": str(cls.get(t)), - "newline": os.linesep} - s += "{label} {start} {end} {elapsed} {status} {newline}".format( - **data) + data = { + "label": t, + "start": str(cls.timer_start[t]), + "end": str(cls.timer_end[t]), + "status": str(cls.timer_status[t]), + "elapsed": str(cls.get(t)), + "newline": os.linesep, + } + s += "{label} {start} {end} {elapsed} {status} {newline}".format(**data) return s @classmethod @@ -514,26 +518,23 @@ def systeminfo(cls, data=None): if data is not None: data_platform.update(data) return Printer.attribute( - data_platform, - order=["Machine Attribute", "Value"], - output="table" + data_platform, order=["Machine Attribute", "Value"], output="table" ) @classmethod - def get_sysinfo(cls, - node=None, - user=None): + def get_sysinfo(cls, node=None, user=None): data_platform = cm_systeminfo(node=node, user=user) return data_platform @classmethod - def get_benchmark(cls, - sysinfo=True, - tag=None, - node=None, - user=None, - total=False, - ): + def get_benchmark( + cls, + sysinfo=True, + tag=None, + node=None, + user=None, + total=False, + ): """prints out all timers in a convenient benchmark table Args: @@ -557,13 +558,10 @@ def get_benchmark(cls, data_platform = cm_systeminfo(user=user, node=node) if sysinfo: - print(Printer.attribute( - data_platform, - output="table" - )) + print(Printer.attribute(data_platform, output="table")) benchmark_data = { - 'sysinfo': data_platform, + "sysinfo": data_platform, } # @@ -572,35 +570,37 @@ def get_benchmark(cls, timers = StopWatch.keys() total_time = 0.0 if len(timers) > 0: - data_timers = {} for timer in timers: data_timers[timer] = { - 'start': time.strftime("%Y-%m-%d %H:%M:%S", - time.gmtime( - StopWatch.timer_start[timer])), - 'stop': time.strftime("%Y-%m-%d %H:%M:%S", - time.gmtime( - StopWatch.timer_end[timer])), - 'time': StopWatch.get(timer, digits=3), - 'sum': StopWatch.sum(timer, digits=3), - 'status': StopWatch.get_status(timer), - 'msg': StopWatch.get_message(timer), - 'timer': timer, - 'tag': tag or '' + "start": time.strftime( + "%Y-%m-%d %H:%M:%S", time.gmtime(StopWatch.timer_start[timer]) + ), + "stop": time.strftime( + "%Y-%m-%d %H:%M:%S", time.gmtime(StopWatch.timer_end[timer]) + ), + "time": StopWatch.get(timer, digits=3), + "sum": StopWatch.sum(timer, digits=3), + "status": StopWatch.get_status(timer), + "msg": StopWatch.get_message(timer), + "timer": timer, + "tag": tag or "", } total_time = total_time + StopWatch.get(timer) # print(Printer.attribute(data_timers, header=["Command", "Time/s"])) - if 'benchmark_start_stop' in data_timers: - del data_timers['benchmark_start_stop'] + if "benchmark_start_stop" in data_timers: + del data_timers["benchmark_start_stop"] for key in data_timers: - if key != 'benchmark_start_stop' and data_timers[key]['status'] is None: - data_timers[key]['status'] = "failed" - elif data_timers[key]['status'] is not None and data_timers[key]['status']: - data_timers[key]['status'] = "ok" + if key != "benchmark_start_stop" and data_timers[key]["status"] is None: + data_timers[key]["status"] = "failed" + elif ( + data_timers[key]["status"] is not None + and data_timers[key]["status"] + ): + data_timers[key]["status"] = "ok" if total: print("Total:", total_time) @@ -613,19 +613,21 @@ def get_benchmark(cls, return benchmark_data @classmethod - def benchmark(cls, - sysinfo=True, - timers=True, - csv=True, - prefix="# csv", - tag=None, - sum=True, - node=None, - user=None, - version=None, - attributes=None, - total=False, - filename=None): + def benchmark( + cls, + sysinfo=True, + timers=True, + csv=True, + prefix="# csv", + tag=None, + sum=True, + node=None, + user=None, + version=None, + attributes=None, + total=False, + filename=None, + ): """prints out all timers in a convenient benchmark table Args: @@ -651,44 +653,43 @@ def benchmark(cls, data_platform = cm_systeminfo(user=user, node=node) if sysinfo: content = content + Printer.attribute( - data_platform, - order=["Machine Attribute", "Value"], - output="table" + data_platform, order=["Machine Attribute", "Value"], output="table" ) content = content + "\n" if timers: - # # PRINT TIMERS # timers = StopWatch.keys() total_time = 0.0 if len(timers) > 0: - data_timers = {} for timer in timers: data_timers[timer] = { - 'start': time.strftime("%Y-%m-%d %H:%M:%S", - time.gmtime( - StopWatch.timer_start[timer])), - 'time': StopWatch.get(timer, digits=3), - 'sum': StopWatch.sum(timer, digits=3), - 'status': StopWatch.get_status(timer), - 'msg': StopWatch.get_message(timer), - 'timer': timer, - 'tag': tag or '' + "start": time.strftime( + "%Y-%m-%d %H:%M:%S", + time.gmtime(StopWatch.timer_start[timer]), + ), + "time": StopWatch.get(timer, digits=3), + "sum": StopWatch.sum(timer, digits=3), + "status": StopWatch.get_status(timer), + "msg": StopWatch.get_message(timer), + "timer": timer, + "tag": tag or "", } try: total_time = total_time + StopWatch.get(timer) except: # noqa: E722 pass - for attribute in ["uname.node", - "user", - "uname.system", - "uname.machine", - "platform.version", - "sys.platform"]: + for attribute in [ + "uname.node", + "user", + "uname.system", + "uname.machine", + "platform.version", + "sys.platform", + ]: if attribute == "user" and user is not None: data_timers[timer][attribute] = user elif attribute == "uname.node" and node is not None: @@ -701,14 +702,20 @@ def benchmark(cls, # print(Printer.attribute(data_timers, header=["Command", "Time/s"])) - if 'benchmark_start_stop' in data_timers: - del data_timers['benchmark_start_stop'] + if "benchmark_start_stop" in data_timers: + del data_timers["benchmark_start_stop"] for key in data_timers: - if key != 'benchmark_start_stop' and data_timers[key]['status'] is None: - data_timers[key]['status'] = "failed" - elif data_timers[key]['status'] is not None and data_timers[key]['status']: - data_timers[key]['status'] = "ok" + if ( + key != "benchmark_start_stop" + and data_timers[key]["status"] is None + ): + data_timers[key]["status"] = "failed" + elif ( + data_timers[key]["status"] is not None + and data_timers[key]["status"] + ): + data_timers[key]["status"] = "ok" if attributes is None: order = [ @@ -722,7 +729,7 @@ def benchmark(cls, "uname.node", "user", "uname.system", - "platform.version" + "platform.version", ] header = [ @@ -736,30 +743,18 @@ def benchmark(cls, "Node", "User", "OS", - "Version" + "Version", ] elif attributes == "short": - order = [ - "timer", - "status", - "time" - ] + order = ["timer", "status", "time"] - header = [ - "Name", - "Status", - "Time" - ] + header = ["Name", "Status", "Time"] else: order = attributes header = attributes content = content + "\n" content = content + Printer.write( - data_timers, - order=order, - header=header, - output="table" - + data_timers, order=order, header=header, output="table" ) if total: @@ -775,20 +770,14 @@ def benchmark(cls, order = ["# csv"] + order content = content + Printer.write( - data_timers, - order=order, - header=header, - output="csv" + data_timers, order=order, header=header, output="csv" ) else: - content = content + pprint.pformat(data_timers, indent=4) content = content + "\n" content = content + Printer.write( - data_timers, - order=order[1:], - output="csv" + data_timers, order=order[1:], output="csv" ) content = content + "\n" @@ -799,19 +788,24 @@ def benchmark(cls, if filename: writefile(filename, content) - def load(filename, - label=["name"], label_split_char=" ", - attributes=['timer', - 'status', - 'time', - 'sum', - 'start', - 'tag', - 'msg', - 'uname.node', - 'user', - 'uname.system', - 'platform.version']): + def load( + filename, + label=["name"], + label_split_char=" ", + attributes=[ + "timer", + "status", + "time", + "sum", + "start", + "tag", + "msg", + "uname.node", + "user", + "uname.system", + "platform.version", + ], + ): """Loads data written to a file from the #csv lines. If the timer name has spaces in it, it must also have a label tag in which each lable is the name when splitting up the timer name. The list of attributes is the list specified plus the once generated from the @@ -828,6 +822,7 @@ def load(filename, """ from cloudmesh.common.Shell import Shell + data = [] headers = [] content = readfile(filename) @@ -846,12 +841,10 @@ def load(filename, entry = entry + label_tags data.append(entry) - return {"headers": headers, - "data": data} + return {"headers": headers, "data": data} class StopWatchBlock: - def __init__(self, name, data=None, log=sys.stdout, mode="w"): self.name = name self.data = data @@ -871,7 +864,10 @@ def __exit__(self, type, value, traceback): StopWatch.stop(self.name) entry = StopWatch.get(self.name) if self.data: - print(f"# {self.name}, {entry}, {self.start}, {self.stop}, {self.data}", file=self.log) + print( + f"# {self.name}, {entry}, {self.start}, {self.stop}, {self.data}", + file=self.log, + ) else: print(f"# {self.name}, {entry}, {self.start}, {self.stop}", file=self.log) if self.is_file: diff --git a/src/cloudmesh/common/Tabulate.py b/src/cloudmesh/common/Tabulate.py index 5846dd5..33c0eea 100644 --- a/src/cloudmesh/common/Tabulate.py +++ b/src/cloudmesh/common/Tabulate.py @@ -85,8 +85,9 @@ def select(results, order=None, width=None): field = result[key] if type(field) == list: field = ", ".join(field) - entry.append("\n".join( - textwrap.wrap(str(field), _width[i]))) + entry.append( + "\n".join(textwrap.wrap(str(field), _width[i])) + ) except: # noqa: E722 entry.append("") else: @@ -96,15 +97,16 @@ def select(results, order=None, width=None): return _results @staticmethod - def write(results, - order=None, - header=None, - output='table', - width=None, - humanize=False, - max_width=None, - sep='.'): - + def write( + results, + order=None, + header=None, + output="table", + width=None, + humanize=False, + max_width=None, + sep=".", + ): width = width or max_width # maxwidth is deprecated if header: @@ -112,7 +114,7 @@ def write(results, elif order: header = list(order) - if output == 'table': + if output == "table": output = Printer.default() # @@ -124,31 +126,26 @@ def write(results, _results = list(results) if output == "json": - return json.dumps(_results, - # sort_keys=sort_keys, - indent=4) + return json.dumps( + _results, + # sort_keys=sort_keys, + indent=4, + ) elif output == "yaml": - return yaml.dump(convert_from_unicode(_results), - default_flow_style=False) + return yaml.dump(convert_from_unicode(_results), default_flow_style=False) elif output == "dict": return _results - elif output == 'csv': + elif output == "csv": flat = flatten(_results, sep=sep) - return Printer.csv(flat, - order=order, - header=header, - humanize=humanize, - sort_keys=True) - - _results = Printer.select(_results, - order=order, - width=width) + return Printer.csv( + flat, order=order, header=header, humanize=humanize, sort_keys=True + ) - if output in ['flat', 'html']: + _results = Printer.select(_results, order=order, width=width) + if output in ["flat", "html"]: if order is not None: - flat = [] for element in results: _element = {} @@ -159,8 +156,8 @@ def write(results, _results = flat - if output in ['flat']: - return (_results) + if output in ["flat"]: + return _results if header: return tabulate_printer(_results, tablefmt=output, headers=header) @@ -183,17 +180,18 @@ def _to_tabulate(d): return results @staticmethod - def flatwrite(table, - order=None, - header=None, - output="table", - sort_keys=True, - show_none="", - humanize=None, - sep=".", - max_width=48, # deprecated use width - width=48, - ): + def flatwrite( + table, + order=None, + header=None, + output="table", + sort_keys=True, + show_none="", + humanize=None, + sep=".", + max_width=48, # deprecated use width + width=48, + ): """writes the information given in the table Args: @@ -211,16 +209,15 @@ def flatwrite(table, Returns: """ - width = width or max_width # max_width is deprecated. thi is used for those still using it + width = ( + width or max_width + ) # max_width is deprecated. thi is used for those still using it flat = flatten(table, sep=sep) - return Printer.write(flat, - order=order, - header=header, - output=output, - width=width - ) + return Printer.write( + flat, order=order, header=header, output=output, width=width + ) """ return Printer.write(flat, sort_keys=sort_keys, @@ -233,13 +230,16 @@ def flatwrite(table, """ @classmethod - def attribute(cls, d, - header=None, - order=None, - sort_keys=True, - humanize=None, - output="table", - width=70): + def attribute( + cls, + d, + header=None, + order=None, + sort_keys=True, + humanize=None, + output="table", + width=70, + ): """prints a attribute/key value table Args: @@ -278,17 +278,13 @@ def attribute(cls, d, else: x.append([key, str(d[key]) or ""]) - if output == 'table': + if output == "table": output = Printer.default() return tabulate_printer(x, tablefmt=output, headers=header) @classmethod - def csv(cls, d, - order=None, - header=None, - humanize=None, - sort_keys=True): + def csv(cls, d, order=None, header=None, humanize=None, sort_keys=True): """prints a table in csv format Args: @@ -314,7 +310,7 @@ def _get(element, key): try: tmp = str(d[element][key]) except: # noqa: E722 - tmp = ' ' + tmp = " " return tmp if d is None or d == {}: diff --git a/src/cloudmesh/common/console.py b/src/cloudmesh/common/console.py index 78638e3..3f9de30 100644 --- a/src/cloudmesh/common/console.py +++ b/src/cloudmesh/common/console.py @@ -10,12 +10,11 @@ colorama.init() + def is_powershell(): - # this function is better than the one in util # but not using that one since it is a circular import - return len(os.getenv('PSModulePath', '').split(os.pathsep)) >= 3 - + return len(os.getenv("PSModulePath", "").split(os.pathsep)) >= 3 def indent(text, indent=2, width=128): @@ -30,10 +29,13 @@ def indent(text, indent=2, width=128): """ return "\n".join( - textwrap.wrap(text, - width=width, - initial_indent=" " * indent, - subsequent_indent=" " * indent)) + textwrap.wrap( + text, + width=width, + initial_indent=" " * indent, + subsequent_indent=" " * indent, + ) + ) class Console(object): @@ -61,39 +63,39 @@ class Console(object): debug = True theme_color = { - 'HEADER': Fore.MAGENTA, - 'BLACK': Fore.BLACK, - 'CYAN': Fore.CYAN, - 'WHITE': Fore.WHITE, - 'BLUE': Fore.BLUE, - 'OKBLUE': Fore.BLUE, - 'OKGREEN': Fore.GREEN, - 'GREEN': Fore.GREEN, - 'FAIL': Fore.RED, - 'WARNING': Fore.MAGENTA, - 'RED': Fore.RED, - 'ENDC': Style.RESET_ALL, - 'BOLD': Style.BRIGHT, - 'NORMAL': Style.NORMAL + "HEADER": Fore.MAGENTA, + "BLACK": Fore.BLACK, + "CYAN": Fore.CYAN, + "WHITE": Fore.WHITE, + "BLUE": Fore.BLUE, + "OKBLUE": Fore.BLUE, + "OKGREEN": Fore.GREEN, + "GREEN": Fore.GREEN, + "FAIL": Fore.RED, + "WARNING": Fore.MAGENTA, + "RED": Fore.RED, + "ENDC": Style.RESET_ALL, + "BOLD": Style.BRIGHT, + "NORMAL": Style.NORMAL # 'ENDC': '\033[0m', # 'BOLD': "\033[1m", } theme_bw = { - 'HEADER': '', - 'BLACK': '', - 'CYAN': '', - 'WHITE': '', - 'BLUE': '', - 'OKBLUE': '', - 'OKGREEN': '', - 'GREEN': '', - 'FAIL': '', - 'WARNING': '', - 'RED': '', - 'ENDC': '', - 'BOLD': "", - 'NORMAL': "" + "HEADER": "", + "BLACK": "", + "CYAN": "", + "WHITE": "", + "BLUE": "", + "OKBLUE": "", + "OKGREEN": "", + "GREEN": "", + "FAIL": "", + "WARNING": "", + "RED": "", + "ENDC": "", + "BOLD": "", + "NORMAL": "", } theme = theme_color @@ -180,7 +182,7 @@ def get(name): if name in Console.theme: return Console.theme[name] else: - return Console.theme['BLACK'] + return Console.theme["BLACK"] @staticmethod def txt_msg(message, width=79): @@ -241,9 +243,11 @@ def error(cls, message, prefix=True, traceflag=False): text = "" if cls.color: if is_powershell(): - print(Fore.RED + Back.WHITE + text + message + Console.theme_color['ENDC']) + print( + Fore.RED + Back.WHITE + text + message + Console.theme_color["ENDC"] + ) else: - cls.cprint('FAIL', text, str(message)) + cls.cprint("FAIL", text, str(message)) else: print(cls.txt_msg(text + str(message))) @@ -273,7 +277,7 @@ def TODO(message, prefix=True, traceflag=True): else: text = "" if Console.color: - Console.cprint('FAIL', text, str(message)) + Console.cprint("FAIL", text, str(message)) else: print(Console.msg(text + str(message))) @@ -296,9 +300,9 @@ def debug_msg(message): """ message = message or "" if Console.color: - Console.cprint('RED', 'DEBUG: ', message) + Console.cprint("RED", "DEBUG: ", message) else: - print(Console.msg('DEBUG: ' + message)) + print(Console.msg("DEBUG: " + message)) @staticmethod def info(message): @@ -312,7 +316,7 @@ def info(message): """ message = message or "" if Console.color: - Console.cprint('OKBLUE', "INFO: ", message) + Console.cprint("OKBLUE", "INFO: ", message) else: print(Console.msg("INFO: " + message)) @@ -330,7 +334,13 @@ def warning(message): if Console.color: if is_powershell(): # fixes powershell problem https://github.com/nodejs/node/issues/14243 - print(Fore.MAGENTA + Style.BRIGHT + "WARNING: " + message + Console.theme_color['ENDC']) + print( + Fore.MAGENTA + + Style.BRIGHT + + "WARNING: " + + message + + Console.theme_color["ENDC"] + ) else: Console.cprint("WARNING", "WARNING: ", message) else: @@ -348,7 +358,7 @@ def ok(message): """ message = message or "" if Console.color: - Console.cprint('OKGREEN', "", message) + Console.cprint("OKGREEN", "", message) else: print(Console.msg(message)) @@ -367,10 +377,12 @@ def cprint(color="BLUE", prefix="", message=""): message = message or "" prefix = prefix or "" - print(Console.theme_color[color] + prefix + message + Console.theme_color['ENDC']) + print( + Console.theme_color[color] + prefix + message + Console.theme_color["ENDC"] + ) @staticmethod - def text(color='RED', prefix=None, message=None): + def text(color="RED", prefix=None, message=None): """returns a message in a given color Args: @@ -383,7 +395,7 @@ def text(color='RED', prefix=None, message=None): """ message = message or "" prefix = prefix or "" - return (Console.theme[color] + prefix + message + Console.theme['ENDC']) + return Console.theme[color] + prefix + message + Console.theme["ENDC"] # @@ -407,9 +419,9 @@ def text(color='RED', prefix=None, message=None): print(Console.color) Console.error("Error") - print(Fore.RED + 'some red text') - print(Back.GREEN + 'and with a green background') - print(Style.DIM + 'and in dim text') + print(Fore.RED + "some red text") + print(Back.GREEN + "and with a green background") + print(Style.DIM + "and in dim text") print(Fore.RESET + Back.RESET + Style.RESET_ALL) - print('back to normal now') + print("back to normal now") Console.line() diff --git a/src/cloudmesh/common/debug.py b/src/cloudmesh/common/debug.py index 7703c1a..eddf90f 100644 --- a/src/cloudmesh/common/debug.py +++ b/src/cloudmesh/common/debug.py @@ -31,20 +31,27 @@ def tracefunc(frame, event, arg, indent=[0]): # noinspection PyPep8Naming -def VERBOSE(msg, label=None, color="BLUE", verbose=9, location=True, - secrets=["OS_PASSWORD", - "OS_USERNAME", - "client_secret", - "client_id", - "project_id", - "AZURE_TENANT_ID", - "AZURE_SUBSCRIPTION_ID", - "AZURE_APPLICATION_ID", - "AZURE_SECRET_KEY: TBD", - "EC2_ACCESS_ID: TBD", - "EC2_SECRET_KEY", - "MONGO_PASSWORD"] - ): +def VERBOSE( + msg, + label=None, + color="BLUE", + verbose=9, + location=True, + secrets=[ + "OS_PASSWORD", + "OS_USERNAME", + "client_secret", + "client_id", + "project_id", + "AZURE_TENANT_ID", + "AZURE_SUBSCRIPTION_ID", + "AZURE_APPLICATION_ID", + "AZURE_SECRET_KEY: TBD", + "EC2_ACCESS_ID: TBD", + "EC2_SECRET_KEY", + "MONGO_PASSWORD", + ], +): """Prints a data structure in verbose mode Args: @@ -62,7 +69,6 @@ def VERBOSE(msg, label=None, color="BLUE", verbose=9, location=True, _verbose = int(Variables()["verbose"] or 0) if _verbose >= verbose: - verbose_lock.acquire() if label is None: @@ -84,12 +90,12 @@ def VERBOSE(msg, label=None, color="BLUE", verbose=9, location=True, for key in secrets: if key in tmp: tmp[key] = "********" - banner(lineno + " " + filename + hline + pformat(tmp), - label=label, - color=color) + banner( + lineno + " " + filename + hline + pformat(tmp), label=label, color=color + ) else: - banner(lineno + " " + filename + hline + pformat(msg), - label=label, - color=color) + banner( + lineno + " " + filename + hline + pformat(msg), label=label, color=color + ) verbose_lock.release() diff --git a/src/cloudmesh/common/default.py b/src/cloudmesh/common/default.py index 4f6da9d..d97c78c 100644 --- a/src/cloudmesh/common/default.py +++ b/src/cloudmesh/common/default.py @@ -21,12 +21,12 @@ class Default(object): _index(context: str, key: str) -> str: Creates a unique index for each key-value pair in the data dictionary. """ - + def _index(self, context, key): """ Creates a unique index for the data dictionary. - This method combines the context and the key into a single string, separated by a comma. + This method combines the context and the key into a single string, separated by a comma. This string serves as a unique index for each key-value pair in the data dictionary. Args: @@ -112,7 +112,7 @@ def __setitem__(self, context_key, value): """ context, key = context_key self.data[self._index(context, key)] = value - + def __delitem__(self, context_key): """ Deletes a specific key-value pair from the data dictionary. @@ -138,6 +138,7 @@ def __delitem__(self, context_key): print("E", element, context) if element.startswith(context + ","): del self.data[element] + def __contains__(self, item): """ Checks if a specific value exists in the data dictionary. @@ -201,7 +202,7 @@ def __repr__(self): """ Returns a string representation of the data dictionary suitable for debugging. - This method returns a string that, if fed to the eval() function, should produce an equivalent object. + This method returns a string that, if fed to the eval() function, should produce an equivalent object. In this case, it returns a string representation of the data dictionary. Returns: @@ -232,13 +233,14 @@ def close(self): """ Closes the data dictionary. - This method is used to safely close the data dictionary when it is no longer needed, + This method is used to safely close the data dictionary when it is no longer needed, freeing up any resources it was using. Raises: Exception: If the data dictionary cannot be closed. """ # ... existing code ... + def close(self): self.data.close() @@ -266,6 +268,6 @@ def close(self): assert v["chameleon", "bla"] is None print(v["chameleon"]) - assert v["chameleon"]['image'] == 'i_c' + assert v["chameleon"]["image"] == "i_c" print(v["bla"]) assert v["bla"] is None diff --git a/src/cloudmesh/common/deprecated.py b/src/cloudmesh/common/deprecated.py index 7c7ff59..4ae1ff1 100644 --- a/src/cloudmesh/common/deprecated.py +++ b/src/cloudmesh/common/deprecated.py @@ -6,7 +6,7 @@ from cloudmesh.common.console import Console -string_types = (type(b''), type(u'')) +string_types = (type(b""), type("")) def deprecated(reason): @@ -17,7 +17,6 @@ def deprecated(reason): """ if isinstance(reason, string_types): - # The @deprecated is used with a 'reason'. # # .. code-block:: python @@ -27,7 +26,6 @@ def deprecated(reason): # pass def decorator(func1): - if inspect.isclass(func1): fmt1 = "Call to deprecated class {name} ({reason})." else: @@ -35,13 +33,13 @@ def decorator(func1): @functools.wraps(func1) def new_func1(*args, **kwargs): - warnings.simplefilter('always', DeprecationWarning) + warnings.simplefilter("always", DeprecationWarning) warnings.warn( fmt1.format(name=func1.__name__, reason=reason), category=DeprecationWarning, - stacklevel=2 + stacklevel=2, ) - warnings.simplefilter('default', DeprecationWarning) + warnings.simplefilter("default", DeprecationWarning) Console.warning(fmt1.format(name=func1.__name__, reason=reason)) return func1(*args, **kwargs) @@ -50,7 +48,6 @@ def new_func1(*args, **kwargs): return decorator elif inspect.isclass(reason) or inspect.isfunction(reason): - # The @deprecated is used without any 'reason'. # # .. code-block:: python @@ -68,13 +65,13 @@ def new_func1(*args, **kwargs): @functools.wraps(func2) def new_func2(*args, **kwargs): - warnings.simplefilter('always', DeprecationWarning) + warnings.simplefilter("always", DeprecationWarning) warnings.warn( fmt2.format(name=func2.__name__), category=DeprecationWarning, - stacklevel=2 + stacklevel=2, ) - warnings.simplefilter('default', DeprecationWarning) + warnings.simplefilter("default", DeprecationWarning) Console.warning(fmt2.format(name=func2.__name__, reason=reason)) return func2(*args, **kwargs) diff --git a/src/cloudmesh/common/location.py b/src/cloudmesh/common/location.py index aed7f6a..d822885 100644 --- a/src/cloudmesh/common/location.py +++ b/src/cloudmesh/common/location.py @@ -9,6 +9,7 @@ from cloudmesh.common.util import readfile from cloudmesh.common.util import writefile + class Location: _shared_state = None diff --git a/src/cloudmesh/common/logger.py b/src/cloudmesh/common/logger.py index 127001f..7739927 100644 --- a/src/cloudmesh/common/logger.py +++ b/src/cloudmesh/common/logger.py @@ -30,8 +30,13 @@ def LOGGER(filename): try: location = Location() - level = grep("loglevel:", location.file("cloudmesh_debug.yaml")) \ - .strip().split(":")[1].strip().lower() + level = ( + grep("loglevel:", location.file("cloudmesh_debug.yaml")) + .strip() + .split(":")[1] + .strip() + .lower() + ) if level.upper() == "DEBUG": loglevel = logging.DEBUG @@ -51,7 +56,8 @@ def LOGGER(filename): log.setLevel(loglevel) formatter = logging.Formatter( - 'CM {0:>50}:%(lineno)s: %(levelname)6s - %(message)s'.format(name)) + "CM {0:>50}:%(lineno)s: %(levelname)6s - %(message)s".format(name) + ) # formatter = logging.Formatter( # 'CM {0:>50}: %(levelname)6s - %(module)s:%(lineno)s %funcName)s: %(message)s'.format(name)) diff --git a/src/cloudmesh/common/parameter.py b/src/cloudmesh/common/parameter.py index c077d6b..25a5d2a 100644 --- a/src/cloudmesh/common/parameter.py +++ b/src/cloudmesh/common/parameter.py @@ -3,7 +3,6 @@ class Parameter(object): - @staticmethod def parse(arguments, **kwargs): """parses arguments based on their kind and aplies Parameter.expand or @@ -32,13 +31,13 @@ def parse(arguments, **kwargs): result = dotdict(arguments) if kwargs is not None: for k, value in kwargs.items(): - if value == 'expand': + if value == "expand": result[k] = Parameter.expand(result[k]) - elif value == 'dict': + elif value == "dict": result[k] = Parameter.expand(result[k]) result[k] = Parameter.arguments_to_dict(result[k]) - elif value == 'str': - result[k] = ' '.join(result[k]) + elif value == "str": + result[k] = " ".join(result[k]) return result @staticmethod @@ -55,13 +54,13 @@ def _expand(values): if "," in values: found = values.split(",") elif "-" in values: - _from, _to = values.split('-') + _from, _to = values.split("-") upper = [chr(x) for x in range(65, 91)] lower = [chr(x) for x in range(97, 123)] all = upper + lower i_start = all.index(_from) i_end = all.index(_to) - found = all[i_start:i_end + 1] # not sure why there is +1 + found = all[i_start : i_end + 1] # not sure why there is +1 else: found = [values] return found @@ -85,7 +84,6 @@ def expand_string(cls, parameter): return [parameter] elif "[" in parameter: - prefix, found = parameter.split("[", 1) found, postfix = found.split("]", 1) @@ -119,9 +117,9 @@ def expand(cls, parameter, allow_duplicates=False, sort=False, sep=":"): if type(parameter) == list or parameter is None: return parameter - parameters = list(expand_hostlist(parameter, - allow_duplicates=False, - sort=False)) + parameters = list( + expand_hostlist(parameter, allow_duplicates=False, sort=False) + ) results = [t.split(sep, 1) for t in parameters] merge = [] @@ -133,7 +131,6 @@ def expand(cls, parameter, allow_duplicates=False, sort=False, sep=":"): return parameters elif len(merge) == len(parameters) + 1 and len(results[0]) == 2: - prefix = results[0][0] _results = [] for i in range(1, len(parameters)): @@ -142,7 +139,6 @@ def expand(cls, parameter, allow_duplicates=False, sort=False, sep=":"): return parameters else: - return parameters @staticmethod @@ -179,11 +175,11 @@ def find_bool(name, *dicts): for d in dicts: if type(d) == str: - value = d == 'True' + value = d == "True" elif name in d: value = d[name] if type(value) == str: - value = value == 'True' + value = value == "True" if value: return True diff --git a/src/cloudmesh/common/prettytable.py b/src/cloudmesh/common/prettytable.py index 186ff9c..52e464e 100644 --- a/src/cloudmesh/common/prettytable.py +++ b/src/cloudmesh/common/prettytable.py @@ -54,7 +54,7 @@ else: itermap = itertools.imap iterzip = itertools.izip - uni_chr = chr # unichr was for old versions of python + uni_chr = chr # unichr was for old versions of python from HTMLParser import HTMLParser if py3k and sys.version_info[1] >= 2: @@ -78,7 +78,7 @@ def _get_size(text): lines = text.splitlines() height = len(lines) - if (len(lines) > 0): + if len(lines) > 0: width = max([_str_block_width(line) for line in lines]) else: width = 0 @@ -86,9 +86,7 @@ def _get_size(text): class PrettyTable(object): - def __init__(self, field_names=None, **kwargs): - """Return a new PrettyTable instance Arguments: @@ -132,8 +130,12 @@ def __init__(self, field_names=None, **kwargs): # Options self._options = "start end fields header border sortby reversesort sort_key attributes format hrules vrules".split() - self._options.extend("int_format float_format padding_width left_padding_width right_padding_width".split()) - self._options.extend("vertical_char horizontal_char junction_char header_style valign".split()) + self._options.extend( + "int_format float_format padding_width left_padding_width right_padding_width".split() + ) + self._options.extend( + "vertical_char horizontal_char junction_char header_style valign".split() + ) for option in self._options: if option in kwargs: self._validate_option(option, kwargs[option]) @@ -205,7 +207,6 @@ def _justify(self, text, width, align): return (excess // 2) * " " + text + (excess // 2) * " " def __getattr__(self, name): - if name == "rowcount": return len(self._rows) elif name == "colcount": @@ -219,7 +220,6 @@ def __getattr__(self, name): raise AttributeError(name) def __getitem__(self, index): - new = PrettyTable() new.field_names = self.field_names for attr in self._options: @@ -231,13 +231,18 @@ def __getitem__(self, index): elif isinstance(index, int): new.add_row(self._rows[index]) else: - raise Exception("Index %s is invalid, must be an integer or slice" % str(index)) + raise Exception( + "Index %s is invalid, must be an integer or slice" % str(index) + ) return new if py3k: + def __str__(self): return self.__unicode__() + else: + def __str__(self): return self.__unicode__().encode(self.encoding) @@ -258,8 +263,15 @@ def __unicode__(self): def _validate_option(self, option, val): if option in ("field_names"): self._validate_field_names(val) - elif option in ("start", "end", "max_width", "padding_width", - "left_padding_width", "right_padding_width", "format"): + elif option in ( + "start", + "end", + "max_width", + "padding_width", + "left_padding_width", + "right_padding_width", + "format", + ): self._validate_nonnegative_int(option, val) elif option in ("sortby"): self._validate_field_name(option, val) @@ -292,14 +304,19 @@ def _validate_field_names(self, val): try: assert len(val) == len(self._field_names) except AssertionError: - raise Exception("Field name list has incorrect number " - "of values, (actual) %d!=%d (expected)" % (len(val), len(self._field_names))) + raise Exception( + "Field name list has incorrect number " + "of values, (actual) %d!=%d (expected)" + % (len(val), len(self._field_names)) + ) if self._rows: try: assert len(val) == len(self._rows[0]) except AssertionError: - raise Exception("Field name list has incorrect number of values," - " (actual) %d!=%d (expected)" % (len(val), len(self._rows[0]))) + raise Exception( + "Field name list has incorrect number of values," + " (actual) %d!=%d (expected)" % (len(val), len(self._rows[0])) + ) # Check for uniqueness try: assert len(val) == len(set(val)) @@ -310,7 +327,9 @@ def _validate_header_style(self, val): try: assert val in ("cap", "title", "upper", "lower", None) except AssertionError: - raise Exception("Invalid header style, use cap, title, upper, lower or None!") + raise Exception( + "Invalid header style, use cap, title, upper, lower or None!" + ) def _validate_align(self, val): try: @@ -343,7 +362,9 @@ def _validate_int_format(self, name, val): assert type(val) in (str, unicode) assert val.isdigit() except AssertionError: - raise Exception("Invalid value for %s! Must be an integer format string." % name) + raise Exception( + "Invalid value for %s! Must be an integer format string." % name + ) def _validate_float_format(self, name, val): if val == "": @@ -356,7 +377,9 @@ def _validate_float_format(self, name, val): assert bits[0] == "" or bits[0].isdigit() assert bits[1] == "" or bits[1].isdigit() except AssertionError: - raise Exception("Invalid value for %s! Must be a float format string." % name) + raise Exception( + "Invalid value for %s! Must be a float format string." % name + ) def _validate_function(self, name, val): try: @@ -368,13 +391,17 @@ def _validate_hrules(self, name, val): try: assert val in (ALL, FRAME, HEADER, NONE) except AssertionError: - raise Exception("Invalid value for %s! Must be ALL, FRAME, HEADER or NONE." % name) + raise Exception( + "Invalid value for %s! Must be ALL, FRAME, HEADER or NONE." % name + ) def _validate_vrules(self, name, val): try: assert val in (ALL, FRAME, NONE) except AssertionError: - raise Exception("Invalid value for %s! Must be ALL, FRAME, or NONE." % name) + raise Exception( + "Invalid value for %s! Must be ALL, FRAME, or NONE." % name + ) def _validate_field_name(self, name, val): try: @@ -393,7 +420,9 @@ def _validate_single_char(self, name, val): try: assert _str_block_width(val) == 1 except AssertionError: - raise Exception("Invalid value for %s! Must be a string of length 1." % name) + raise Exception( + "Invalid value for %s! Must be a string of length 1." % name + ) def _validate_attributes(self, name, val): try: @@ -531,7 +560,8 @@ def _get_reversesort(self): Arguments: - reveresort - set to True to sort by descending order, or False to sort by ascending order""" + reveresort - set to True to sort by descending order, or False to sort by ascending order + """ return self._reversesort def _set_reversesort(self, val): @@ -545,7 +575,8 @@ def _get_sort_key(self): Arguments: - sort_key - a function which takes one argument and returns something to be sorted""" + sort_key - a function which takes one argument and returns something to be sorted + """ return self._sort_key def _set_sort_key(self, val): @@ -573,7 +604,8 @@ def _get_header_style(self): Arguments: - header_style - stylisation to apply to field names in header ("cap", "title", "upper", "lower" or None)""" + header_style - stylisation to apply to field names in header ("cap", "title", "upper", "lower" or None) + """ return self._header_style def _set_header_style(self, val): @@ -772,7 +804,6 @@ def _set_attributes(self, val): ############################## def _get_options(self, kwargs): - options = {} for option in self._options: if option in kwargs: @@ -787,7 +818,6 @@ def _get_options(self, kwargs): ############################## def set_style(self, style): - if style == DEFAULT: self._set_default_style() elif style == MSWORD_FRIENDLY: @@ -800,7 +830,6 @@ def set_style(self, style): raise Exception("Invalid pre-set style!") def _set_default_style(self): - self.header = True self.border = True self._hrules = FRAME @@ -813,7 +842,6 @@ def _set_default_style(self): self.junction_char = "+" def _set_msword_style(self): - self.header = True self.border = True self._hrules = NONE @@ -823,7 +851,6 @@ def _set_msword_style(self): self.vertical_char = "|" def _set_columns_style(self): - self.header = True self.border = False self.padding_width = 1 @@ -831,7 +858,6 @@ def _set_columns_style(self): self.right_padding_width = 8 def _set_random_style(self): - # Just for fun! self.header = random.choice((True, False)) self.border = random.choice((True, False)) @@ -851,7 +877,6 @@ def _set_random_style(self): ############################## def add_row(self, row): - """Add a row to the table Arguments: @@ -862,13 +887,14 @@ def add_row(self, row): if self._field_names and len(row) != len(self._field_names): raise Exception( - "Row has incorrect number of values, (actual) %d!=%d (expected)" % (len(row), len(self._field_names))) + "Row has incorrect number of values, (actual) %d!=%d (expected)" + % (len(row), len(self._field_names)) + ) if not self._field_names: self.field_names = [("Field %d" % (n + 1)) for n in range(0, len(row))] self._rows.append(list(row)) def del_row(self, row_index): - """Delete a row to the table Arguments: @@ -877,11 +903,13 @@ def del_row(self, row_index): """ if row_index > len(self._rows) - 1: - raise Exception("Cant delete row at index %d, table only has %d rows!" % (row_index, len(self._rows))) + raise Exception( + "Cant delete row at index %d, table only has %d rows!" + % (row_index, len(self._rows)) + ) del self._rows[row_index] def add_column(self, fieldname, column, align="c", valign="t"): - """Add a column to the table. Arguments: @@ -904,16 +932,17 @@ def add_column(self, fieldname, column, align="c", valign="t"): self._rows.append([]) self._rows[i].append(column[i]) else: - raise Exception("Column length %d does not match number of rows %d!" % (len(column), len(self._rows))) + raise Exception( + "Column length %d does not match number of rows %d!" + % (len(column), len(self._rows)) + ) def clear_rows(self): - """Delete all rows from the table but keep the current field names""" self._rows = [] def clear(self): - """Delete all rows and field names from the table, maintaining nothing but styling options""" self._rows = [] @@ -947,13 +976,15 @@ def _compute_widths(self, rows, options): for index, value in enumerate(row): fieldname = self.field_names[index] if fieldname in self.max_width: - widths[index] = max(widths[index], min(_get_size(value)[0], self.max_width[fieldname])) + widths[index] = max( + widths[index], + min(_get_size(value)[0], self.max_width[fieldname]), + ) else: widths[index] = max(widths[index], _get_size(value)[0]) self._widths = widths def _get_padding_widths(self, options): - if options["left_padding_width"] is not None: lpad = options["left_padding_width"] else: @@ -973,7 +1004,7 @@ def _get_rows(self, options): """ # Make a copy of only those rows in the slice range - rows = copy.deepcopy(self._rows[options["start"]:options["end"]]) + rows = copy.deepcopy(self._rows[options["start"] : options["end"]]) # Sort if necessary if options["sortby"]: sortindex = self._field_names.index(options["sortby"]) @@ -986,7 +1017,10 @@ def _get_rows(self, options): return rows def _format_row(self, row, options): - return [self._format_value(field, value) for (field, value) in zip(self._field_names, row)] + return [ + self._format_value(field, value) + for (field, value) in zip(self._field_names, row) + ] def _format_rows(self, rows, options): return [self._format_row(row, options) for row in rows] @@ -996,7 +1030,6 @@ def _format_rows(self, rows, options): ############################## def get_string(self, **kwargs): - """Return string representation of table in current state. Arguments: @@ -1057,7 +1090,6 @@ def get_string(self, **kwargs): return self._unicode("\n").join(lines) def _stringify_hrule(self, options): - if not options["border"]: return "" lpad, rpad = self._get_padding_widths(options) @@ -1070,7 +1102,6 @@ def _stringify_hrule(self, options): return "".join(bits) def _stringify_header(self, options): - bits = [] lpad, rpad = self._get_padding_widths(options) if options["border"]: @@ -1081,7 +1112,10 @@ def _stringify_header(self, options): bits.append(options["vertical_char"]) else: bits.append(" ") - for field, width, in zip(self._field_names, self._widths): + for ( + field, + width, + ) in zip(self._field_names, self._widths): if options["fields"] and field not in options["fields"]: continue if self._header_style == "cap": @@ -1094,7 +1128,11 @@ def _stringify_header(self, options): fieldname = field.lower() else: fieldname = field - bits.append(" " * lpad + self._justify(fieldname, width, self._align[field]) + " " * rpad) + bits.append( + " " * lpad + + self._justify(fieldname, width, self._align[field]) + + " " * rpad + ) if options["border"]: if options["vrules"] == ALL: bits.append(options["vertical_char"]) @@ -1111,8 +1149,12 @@ def _stringify_header(self, options): return "".join(bits) def _stringify_row(self, row, options): - - for index, field, value, width, in zip(range(0, len(row)), self._field_names, row, self._widths): + for ( + index, + field, + value, + width, + ) in zip(range(0, len(row)), self._field_names, row, self._widths): # Enforce max widths lines = value.splitlines() new_lines = [] @@ -1140,14 +1182,21 @@ def _stringify_row(self, row, options): else: bits[y].append(" ") - for field, value, width, in zip(self._field_names, row, self._widths): - + for ( + field, + value, + width, + ) in zip(self._field_names, row, self._widths): valign = self._valign[field] lines = value.splitlines() dHeight = row_height - len(lines) if dHeight: if valign == "m": - lines = [""] * int(dHeight / 2) + lines + [""] * (dHeight - int(dHeight / 2)) + lines = ( + [""] * int(dHeight / 2) + + lines + + [""] * (dHeight - int(dHeight / 2)) + ) elif valign == "b": lines = [""] * dHeight + lines else: @@ -1158,7 +1207,11 @@ def _stringify_row(self, row, options): if options["fields"] and field not in options["fields"]: continue - bits[y].append(" " * lpad + self._justify(l, width, self._align[field]) + " " * rpad) + bits[y].append( + " " * lpad + + self._justify(l, width, self._align[field]) + + " " * rpad + ) if options["border"]: if options["vrules"] == ALL: bits[y].append(self.vertical_char) @@ -1187,7 +1240,6 @@ def _stringify_row(self, row, options): ############################## def get_html_string(self, **kwargs): - """Return string representation of HTML formatted version of table in current state. Arguments: @@ -1219,14 +1271,15 @@ def get_html_string(self, **kwargs): return string def _get_simple_html_string(self, options): - lines = [] open_tag = [] open_tag.append("") lines.append("".join(open_tag)) @@ -1236,7 +1289,9 @@ def _get_simple_html_string(self, options): for field in self._field_names: if options["fields"] and field not in options["fields"]: continue - lines.append(" %s" % escape(field).replace("\n", "
")) + lines.append( + " %s" % escape(field).replace("\n", "
") + ) lines.append(" ") # Data @@ -1247,7 +1302,9 @@ def _get_simple_html_string(self, options): for field, datum in zip(self._field_names, row): if options["fields"] and field not in options["fields"]: continue - lines.append(" %s" % escape(datum).replace("\n", "
")) + lines.append( + " %s" % escape(datum).replace("\n", "
") + ) lines.append(" ") lines.append("") @@ -1255,7 +1312,6 @@ def _get_simple_html_string(self, options): return self._unicode("\n").join(lines) def _get_formatted_html_string(self, options): - lines = [] lpad, rpad = self._get_padding_widths(options) @@ -1263,22 +1319,24 @@ def _get_formatted_html_string(self, options): open_tag.append("") lines.append("".join(open_tag)) @@ -1288,8 +1346,10 @@ def _get_formatted_html_string(self, options): for field in self._field_names: if options["fields"] and field not in options["fields"]: continue - lines.append(" %s" % ( - lpad, rpad, escape(field).replace("\n", "
"))) + lines.append( + ' %s' + % (lpad, rpad, escape(field).replace("\n", "
")) + ) lines.append(" ") # Data @@ -1298,16 +1358,23 @@ def _get_formatted_html_string(self, options): aligns = [] valigns = [] for field in self._field_names: - aligns.append({"l": "left", "r": "right", "c": "center"}[self._align[field]]) - valigns.append({"t": "top", "m": "middle", "b": "bottom"}[self._valign[field]]) + aligns.append( + {"l": "left", "r": "right", "c": "center"}[self._align[field]] + ) + valigns.append( + {"t": "top", "m": "middle", "b": "bottom"}[self._valign[field]] + ) for row in formatted_rows: lines.append(" ") - for field, datum, align, valign in zip(self._field_names, row, aligns, valigns): + for field, datum, align, valign in zip( + self._field_names, row, aligns, valigns + ): if options["fields"] and field not in options["fields"]: continue lines.append( - " %s" % ( - lpad, rpad, align, valign, escape(datum).replace("\n", "
"))) + ' %s' + % (lpad, rpad, align, valign, escape(datum).replace("\n", "
")) + ) lines.append(" ") lines.append("") @@ -1318,35 +1385,36 @@ def _get_formatted_html_string(self, options): # UNICODE WIDTH FUNCTIONS # ############################## + def _char_block_width(char): # Basic Latin, which is probably the most common case # if char in xrange(0x0021, 0x007e): # if char >= 0x0021 and char <= 0x007e: - if 0x0021 <= char <= 0x007e: + if 0x0021 <= char <= 0x007E: return 1 # Chinese, Japanese, Korean (common) - if 0x4e00 <= char <= 0x9fff: + if 0x4E00 <= char <= 0x9FFF: return 2 # Hangul - if 0xac00 <= char <= 0xd7af: + if 0xAC00 <= char <= 0xD7AF: return 2 # Combining? if unicodedata.combining(uni_chr(char)): return 0 # Hiragana and Katakana - if 0x3040 <= char <= 0x309f or 0x30a0 <= char <= 0x30ff: + if 0x3040 <= char <= 0x309F or 0x30A0 <= char <= 0x30FF: return 2 # Full-width Latin characters - if 0xff01 <= char <= 0xff60: + if 0xFF01 <= char <= 0xFF60: return 2 # CJK punctuation - if 0x3000 <= char <= 0x303e: + if 0x3000 <= char <= 0x303E: return 2 # Backspace and delete - if char in (0x0008, 0x007f): + if char in (0x0008, 0x007F): return -1 # Other control characters - elif char in (0x0000, 0x001f): + elif char in (0x0000, 0x001F): return 0 # Take a guess return 1 @@ -1360,6 +1428,7 @@ def _str_block_width(val): # TABLE FACTORIES # ############################## + def from_csv(fp, field_names=None, **kwargs): dialect = csv.Sniffer().sniff(fp.read(1024)) fp.seek(0) @@ -1390,7 +1459,6 @@ def from_db_cursor(cursor, **kwargs): class TableHandler(HTMLParser): - def __init__(self, **kwargs): HTMLParser.__init__(self) self.kwargs = kwargs @@ -1412,8 +1480,7 @@ def handle_endtag(self, tag): stripped_content = self.last_content.strip() self.last_row.append(stripped_content) if tag == "tr": - self.rows.append( - (self.last_row, self.is_last_row_header)) + self.rows.append((self.last_row, self.is_last_row_header)) self.max_row_width = max(self.max_row_width, len(self.last_row)) self.last_row = [] self.is_last_row_header = False @@ -1470,7 +1537,9 @@ def from_html_one(html_code, **kwargs): try: assert len(tables) == 1 except AssertionError: - raise Exception("More than one in provided HTML code! Use from_html instead.") + raise Exception( + "More than one
in provided HTML code! Use from_html instead." + ) return tables[0] @@ -1478,6 +1547,7 @@ def from_html_one(html_code, **kwargs): # MAIN (TEST FUNCTION) # ############################## + def main(): x = PrettyTable(["City name", "Area", "Population", "Annual Rainfall"]) x.sortby = "Population" diff --git a/src/cloudmesh/common/security.py b/src/cloudmesh/common/security.py index 88f17f1..6e20a2d 100644 --- a/src/cloudmesh/common/security.py +++ b/src/cloudmesh/common/security.py @@ -3,6 +3,7 @@ import os import subprocess + def can_use_sudo(): """Checks if the current user has sudo privileges. @@ -11,11 +12,12 @@ def can_use_sudo(): """ try: # The 'id -u' command returns the user ID. When run with sudo, it should return 0 (the ID of the root user). - output = subprocess.check_output('sudo id -u', shell=True).decode().strip() - return output == '0' + output = subprocess.check_output("sudo id -u", shell=True).decode().strip() + return output == "0" except subprocess.CalledProcessError: return False + def generate_strong_pass(): """Generates a password from letters, digits and punctuation diff --git a/src/cloudmesh/common/strdb.py b/src/cloudmesh/common/strdb.py index 62e1af3..6b7c4e9 100644 --- a/src/cloudmesh/common/strdb.py +++ b/src/cloudmesh/common/strdb.py @@ -11,17 +11,21 @@ ############################################################### # make yaml understand unicode + def yaml_construct_unicode(self, node): return self.construct_scalar(node) -yaml.Loader.add_constructor(u'tag:yaml.org,2002:python/unicode', yaml_construct_unicode) -yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:python/unicode', yaml_construct_unicode) +yaml.Loader.add_constructor("tag:yaml.org,2002:python/unicode", yaml_construct_unicode) +yaml.SafeLoader.add_constructor( + "tag:yaml.org,2002:python/unicode", yaml_construct_unicode +) ############################################################### # the db api + class YamlDB(object): """A YAML-backed Key-Value database to store strings""" @@ -35,15 +39,15 @@ def __init__(self, path): os.makedirs(prefix) if os.path.exists(self.path): - with open(self.path, 'rb') as dbfile: + with open(self.path, "rb") as dbfile: self._db = yaml.safe_load(dbfile) or dict() self.flush() def flush(self): string = yaml.dump(self._db, default_flow_style=False) - bits = bytes(string, encoding='utf-8') - with open(self.path, 'wb') as dbfile: + bits = bytes(string, encoding="utf-8") + with open(self.path, "wb") as dbfile: dbfile.write(bits) def __setitem__(self, k, v): diff --git a/src/cloudmesh/common/sudo.py b/src/cloudmesh/common/sudo.py index ce6fd48..f88b1c9 100644 --- a/src/cloudmesh/common/sudo.py +++ b/src/cloudmesh/common/sudo.py @@ -42,6 +42,7 @@ class Sudo: Returns: str: The output created by the write process. """ + @staticmethod def password(msg="sudo password: "): """Prompt the user for the sudo password. @@ -85,8 +86,8 @@ def execute(command, decode="True", debug=False, msg=None): result = subprocess.run(sudo_command, capture_output=True) os.system("sync") if decode: - result.stdout = result.stdout.decode('ascii') - result.stderr = result.stderr.decode('ascii') + result.stdout = result.stdout.decode("ascii") + result.stderr = result.stderr.decode("ascii") if debug: banner("stdout") @@ -137,7 +138,6 @@ def writefile(filename, content, append=False): str: The output created by the write process. """ - os.system("sync") Sudo.password() if append: diff --git a/src/cloudmesh/common/systeminfo.py b/src/cloudmesh/common/systeminfo.py index 3e97e48..a72b388 100644 --- a/src/cloudmesh/common/systeminfo.py +++ b/src/cloudmesh/common/systeminfo.py @@ -30,7 +30,7 @@ def os_is_linux(): bool: True is linux """ try: - content = readfile('/etc/os-release') + content = readfile("/etc/os-release") return platform.system() == "Linux" and "raspbian" not in content except: # noqa: E722 return False @@ -53,7 +53,7 @@ def os_is_pi(): bool: True is Raspberry OS """ try: - content = readfile('/etc/os-release') + content = readfile("/etc/os-release") return platform.system() == "Linux" and "raspbian" in content except: # noqa: E722 return False @@ -63,7 +63,10 @@ def has_window_manager(): if os_is_mac() or os_is_windows(): return True else: - return "GNOME_TERMINAL_SCREEN" in os.environ or "GNOME_TERMINAL_SERVICE" in os.environ + return ( + "GNOME_TERMINAL_SCREEN" in os.environ + or "GNOME_TERMINAL_SERVICE" in os.environ + ) def sys_user(): @@ -103,8 +106,8 @@ def get_platform(): elif sys.platform == "win32": return "windows" try: - content = readfile('/etc/os-release') - if sys.platform == 'linux' and "raspbian" in content: + content = readfile("/etc/os-release") + if sys.platform == "linux" and "raspbian" in content: return "raspberry" else: return sys.platform @@ -160,46 +163,50 @@ def add_binary(value): except: # noqa: E722 pass - data = OrderedDict({ - 'cpu': description.strip(), - 'cpu_count': multiprocessing.cpu_count(), - 'cpu_threads': multiprocessing.cpu_count(), - 'cpu_cores': cores, - 'uname.system': uname.system, - 'uname.node': uname.node, - 'uname.release': uname.release, - 'uname.version': uname.version, - 'uname.machine': uname.machine, - 'uname.processor': uname.processor, - 'sys.platform': sys.platform, - 'python': sys.version, - 'python.version': sys.version.split(" ", 1)[0], - 'python.pip': pip.__version__, - 'user': sys_user(), - 'mem.percent': str(mem.percent) + " %", - 'frequency': frequency - }) - for attribute in ["total", - "available", - "used", - "free", - "active", - "inactive", - "wired" - ]: + data = OrderedDict( + { + "cpu": description.strip(), + "cpu_count": multiprocessing.cpu_count(), + "cpu_threads": multiprocessing.cpu_count(), + "cpu_cores": cores, + "uname.system": uname.system, + "uname.node": uname.node, + "uname.release": uname.release, + "uname.version": uname.version, + "uname.machine": uname.machine, + "uname.processor": uname.processor, + "sys.platform": sys.platform, + "python": sys.version, + "python.version": sys.version.split(" ", 1)[0], + "python.pip": pip.__version__, + "user": sys_user(), + "mem.percent": str(mem.percent) + " %", + "frequency": frequency, + } + ) + for attribute in [ + "total", + "available", + "used", + "free", + "active", + "inactive", + "wired", + ]: try: - data[f"mem.{attribute}"] = \ - humanize.naturalsize(getattr(mem, attribute), binary=True) + data[f"mem.{attribute}"] = humanize.naturalsize( + getattr(mem, attribute), binary=True + ) except: # noqa: E722 pass # svmem(total=17179869184, available=6552825856, percent=61.9, - if data['sys.platform'] == 'darwin': - data['platform.version'] = platform.mac_ver()[0] - elif data['sys.platform'] == 'win32': - data['platform.version'] = platform.win32_ver() + if data["sys.platform"] == "darwin": + data["platform.version"] = platform.mac_ver()[0] + elif data["sys.platform"] == "win32": + data["platform.version"] = platform.win32_ver() else: - data['platform.version'] = uname.version + data["platform.version"] = uname.version try: release_files = Path("/etc").glob("*release") diff --git a/src/cloudmesh/common/util.py b/src/cloudmesh/common/util.py index 9d3d395..eefa40f 100644 --- a/src/cloudmesh/common/util.py +++ b/src/cloudmesh/common/util.py @@ -76,7 +76,7 @@ def exponential_backoff(fn, sleeptime_s_max=30 * 60): if fn(): return True else: - print('Sleeping {} ms'.format(sleeptime_ms)) + print("Sleeping {} ms".format(sleeptime_ms)) time.sleep(sleeptime_ms / 1000.0) sleeptime_ms *= 2 @@ -96,14 +96,13 @@ def download(source, destination, force=False): exists """ if os.path.isfile(destination) and not force: - Console.warning(f"File {destination} already exists. " - "Skipping download ...") + Console.warning(f"File {destination} already exists. " "Skipping download ...") else: - directory = os.path.dirname(destination) Path(directory).mkdir(parents=True, exist_ok=True) r = requests.get(source, allow_redirects=True) - open(destination, 'wb').write(r.content) + open(destination, "wb").write(r.content) + def csv_to_list(csv_string, sep=","): """Converts a CSV table from a string to a list of lists @@ -122,6 +121,7 @@ def csv_to_list(csv_string, sep=","): list_of_lists.append(row) return list_of_lists + def search(lines, pattern): """return all lines that match the pattern #TODO: we need an example @@ -152,7 +152,7 @@ def grep(pattern, filename): # return line return next((L for L in open(filename) if L.find(pattern) >= 0)) except StopIteration: - return '' + return "" def is_local(host): @@ -164,13 +164,14 @@ def is_local(host): Returns: True if the host is the localhost """ - return host in ["127.0.0.1", - "localhost", - socket.gethostname(), - # just in case socket.gethostname() does not work we also try the following: - platform.node(), - socket.gethostbyaddr(socket.gethostname())[0] - ] + return host in [ + "127.0.0.1", + "localhost", + socket.gethostname(), + # just in case socket.gethostname() does not work we also try the following: + platform.node(), + socket.gethostbyaddr(socket.gethostname())[0], + ] # noinspection PyPep8 @@ -181,7 +182,7 @@ def is_gitbash(): True if gitbash """ try: - exepath = os.environ['EXEPATH'] + exepath = os.environ["EXEPATH"] return "Git" in exepath except: # noqa: E722 return False @@ -199,10 +200,12 @@ def is_powershell(): # bash.exe for git bash if platform.system() == "Windows": import psutil - return (psutil.Process(os.getppid()).name() == "powershell.exe") + + return psutil.Process(os.getppid()).name() == "powershell.exe" else: return False + def is_cmd_exe(): """return True if you run in a Windows CMD @@ -213,7 +216,7 @@ def is_cmd_exe(): return False else: try: - return os.environ['OS'] == 'Windows_NT' + return os.environ["OS"] == "Windows_NT" except: # noqa: E722 return False @@ -221,8 +224,8 @@ def is_cmd_exe(): def path_expand(text, slashreplace=True): """returns a string with expanded variable. - :param text: the path to be expanded, which can include ~ and environment variables - :param text: string + :param text: the path to be expanded, which can include ~ and environment variables + :param text: string """ result = os.path.expandvars(os.path.expanduser(text)) @@ -238,25 +241,25 @@ def path_expand(text, slashreplace=True): def convert_from_unicode(data): - """Converts unicode data to a string + """Converts unicode data to a string - Args: - data: the data to convert + Args: + data: the data to convert - Returns: - converted data - """ - if isinstance(data, str): - return str(data) - elif isinstance(data, Mapping): - return dict(map(convert_from_unicode, data.items())) - elif isinstance(data, Iterable): - return type(data)(map(convert_from_unicode, data)) - else: - return data + Returns: + converted data + """ + if isinstance(data, str): + return str(data) + elif isinstance(data, Mapping): + return dict(map(convert_from_unicode, data.items())) + elif isinstance(data, Iterable): + return type(data)(map(convert_from_unicode, data)) + else: + return data -def yn_choice(message, default='y', tries=None): +def yn_choice(message, default="y", tries=None): """asks for a yes/no question. Args: @@ -265,27 +268,35 @@ def yn_choice(message, default='y', tries=None): default: the default answer """ # http://stackoverflow.com/questions/3041986/python-command-line-yes-no-input""" - choices = 'Y/n' if default.lower() in ('y', 'yes') else 'y/N' + choices = "Y/n" if default.lower() in ("y", "yes") else "y/N" if tries is None: choice = input(f"{message} ({choices}) ") - values = ('y', 'yes', '') if default == 'y' else ('y', 'yes') + values = ("y", "yes", "") if default == "y" else ("y", "yes") return True if choice.strip().lower() in values else False else: while tries > 0: choice = input(f"{message} ({choices}) ('q' to discard)") choice = choice.strip().lower() - if choice in ['y', 'yes']: + if choice in ["y", "yes"]: return True - elif choice in ['n', 'no', 'q']: + elif choice in ["n", "no", "q"]: return False else: print("Invalid input...") tries -= 1 -def str_banner(txt=None, c="-", prefix="#", debug=True, label=None, - color="BLUE", padding=False, - figlet=False, font="big"): +def str_banner( + txt=None, + c="-", + prefix="#", + debug=True, + label=None, + color="BLUE", + padding=False, + figlet=False, + font="big", +): """prints a banner of the form with a frame of # around the txt:: # -------------------------- @@ -310,7 +321,6 @@ def str_banner(txt=None, c="-", prefix="#", debug=True, label=None, output += prefix + " " + 70 * c + "\n" if txt is not None: - if figlet: txt = pyfiglet.figlet_format(txt, font=font) @@ -323,9 +333,17 @@ def str_banner(txt=None, c="-", prefix="#", debug=True, label=None, return output -def banner(txt=None, c="-", prefix="#", debug=True, label=None, - color="BLUE", padding=False, - figlet=False, font="big"): +def banner( + txt=None, + c="-", + prefix="#", + debug=True, + label=None, + color="BLUE", + padding=False, + figlet=False, + font="big", +): """prints a banner of the form with a frame of # around the txt:: # -------------------------- @@ -342,8 +360,17 @@ def banner(txt=None, c="-", prefix="#", debug=True, label=None, the banner is larger """ - output = str_banner(txt=txt, c=c, prefix=prefix, debug=debug, label=label, - color=color, padding=padding, figlet=figlet, font=font) + output = str_banner( + txt=txt, + c=c, + prefix=prefix, + debug=debug, + label=label, + color=color, + padding=padding, + figlet=figlet, + font=font, + ) Console.cprint(color, "", output) @@ -417,8 +444,8 @@ def auto_create_version(class_name, version, filename="__init__.py"): """ version_filename = Path( - "{classname}/{filename}".format(classname=class_name, - filename=filename)) + "{classname}/{filename}".format(classname=class_name, filename=filename) + ) with open(version_filename, "r") as f: content = f.read() @@ -443,7 +470,7 @@ def auto_create_requirements(requirements): except: # noqa: E722 file_content = "" - setup_requirements = '\n'.join(requirements) + setup_requirements = "\n".join(requirements) if setup_requirements != file_content: with open("requirements.txt", "w") as text_file: @@ -465,7 +492,7 @@ def copy_files(files_glob, source_dir, dest_dir): shutil.copy2(filename, dest_dir) -def readfile(filename, mode='r', encoding=None): +def readfile(filename, mode="r", encoding=None): """returns the content of a file Args: @@ -477,7 +504,7 @@ def readfile(filename, mode='r', encoding=None): Returns: """ - if mode != 'r' and mode != 'rb': + if mode != "r" and mode != "rb": Console.error(f"incorrect mode : expected 'r' or 'rb' given {mode}") else: content = None @@ -503,12 +530,13 @@ def writefile(filename, content): """ directory = os.path.dirname(filename) - if directory not in [None, '']: + if directory not in [None, ""]: os.makedirs(directory, exist_ok=True) - with open(path_expand(filename), 'w') as outfile: + with open(path_expand(filename), "w") as outfile: outfile.write(content) outfile.truncate() + def appendfile(filename, content): """writes the content into the file @@ -519,11 +547,11 @@ def appendfile(filename, content): Returns: """ - with open(path_expand(filename), 'a') as outfile: + with open(path_expand(filename), "a") as outfile: outfile.write(content) -def writefd(filename, content, mode='w', flags=os.O_RDWR | os.O_CREAT, mask=0o600): +def writefd(filename, content, mode="w", flags=os.O_RDWR | os.O_CREAT, mask=0o600): """writes the content into the file and control permissions Args: @@ -533,7 +561,7 @@ def writefd(filename, content, mode='w', flags=os.O_RDWR | os.O_CREAT, mask=0o60 flags: the os flags that determine the permissions for the file mask: the mask that the permissions will be applied to """ - if mode != 'w' and mode != 'wb': + if mode != "w" and mode != "wb": Console.error(f"incorrect mode : expected 'w' or 'wb' given {mode}") with os.fdopen(os.open(filename, flags, mask), mode) as outfile: @@ -560,7 +588,7 @@ def sudo_readfile(filename, split=True, trim=False): result = result.rstrip() if split: - result = result.split('\n') + result = result.split("\n") return result @@ -603,22 +631,22 @@ def _random_character(texts): def str_bool(value): - return str(value).lower() in ['yes', '1', 'y', 'true', 't'] + return str(value).lower() in ["yes", "1", "y", "true", "t"] def get_password(prompt): from cloudmesh.common.systeminfo import os_is_windows + try: if os_is_windows() and is_gitbash(): continuing = True while continuing: - sys.stdout.write(prompt) sys.stdout.flush() subprocess.check_call(["stty", "-echo"]) password = input() subprocess.check_call(["stty", "echo"]) - sys.stdout.write('Please retype the password:\n') + sys.stdout.write("Please retype the password:\n") sys.stdout.flush() subprocess.check_call(["stty", "-echo"]) password2 = input() @@ -626,24 +654,20 @@ def get_password(prompt): if password == password2: continuing = False else: - Console.error('Passwords do not match\n') + Console.error("Passwords do not match\n") return password else: continuing = True while continuing: password = getpass(prompt) - password2 = getpass('Please retype the password:\n') + password2 = getpass("Please retype the password:\n") if password == password2: continuing = False else: - Console.error('Passwords do not match\n') + Console.error("Passwords do not match\n") return password except KeyboardInterrupt: # Console.error('Detected Ctrl + C. Quitting...') if is_gitbash(): subprocess.check_call(["stty", "echo"]) - raise ValueError('Detected Ctrl + C. Quitting...') - - - - + raise ValueError("Detected Ctrl + C. Quitting...") diff --git a/src/cloudmesh/common/variables.py b/src/cloudmesh/common/variables.py index a99162f..5084971 100644 --- a/src/cloudmesh/common/variables.py +++ b/src/cloudmesh/common/variables.py @@ -8,37 +8,37 @@ class Variables(object): """A class for managing and manipulating variables using a YAML-based storage. - Attributes: - filename (str): The path to the YAML file storing the variables. - data (YamlDB): An instance of the YamlDB class for YAML file manipulation. - - Methods: - save(): Save changes to the YAML file. - get(key, default=None): Retrieve the value associated with a key. - __getitem__(key): Retrieve the value associated with a key using square bracket notation. - __setitem__(key, value): Set the value associated with a key using square bracket notation. - __delitem__(key): Delete the key-value pair associated with the specified key. - __contains__(item): Check if a key exists in the stored data. - __str__(): Return a string representation of the stored data. - __len__(): Return the number of key-value pairs in the stored data. - __add__(directory): Add key-value pairs from a dictionary-like object. - __sub__(keys): Remove key-value pairs for specified keys. - __iter__(): Return an iterator for keys in the stored data. - close(): Close the connection to the YAML file. - clear(): Clear all key-value pairs in the stored data. - dict(): Return the underlying dictionary used for storage. - parameter(attribute, position=0): Retrieve and expand a parameterized value. - boolean(key, value): Set a boolean value based on string representation. - - Examples: - v = Variables() - print(v) - - v["gregor"] = "gregor" - assert "gregor" in v - del v["gregor"] - assert "gregor" not in v - """ + Attributes: + filename (str): The path to the YAML file storing the variables. + data (YamlDB): An instance of the YamlDB class for YAML file manipulation. + + Methods: + save(): Save changes to the YAML file. + get(key, default=None): Retrieve the value associated with a key. + __getitem__(key): Retrieve the value associated with a key using square bracket notation. + __setitem__(key, value): Set the value associated with a key using square bracket notation. + __delitem__(key): Delete the key-value pair associated with the specified key. + __contains__(item): Check if a key exists in the stored data. + __str__(): Return a string representation of the stored data. + __len__(): Return the number of key-value pairs in the stored data. + __add__(directory): Add key-value pairs from a dictionary-like object. + __sub__(keys): Remove key-value pairs for specified keys. + __iter__(): Return an iterator for keys in the stored data. + close(): Close the connection to the YAML file. + clear(): Clear all key-value pairs in the stored data. + dict(): Return the underlying dictionary used for storage. + parameter(attribute, position=0): Retrieve and expand a parameterized value. + boolean(key, value): Set a boolean value based on string representation. + + Examples: + v = Variables() + print(v) + + v["gregor"] = "gregor" + assert "gregor" in v + del v["gregor"] + assert "gregor" not in v + """ def __init__(self, filename=None): """Initialize the Variables instance. @@ -69,7 +69,6 @@ def get(self, key, default=None): """ return self.data.get(key, default) - def __getitem__(self, key): """Retrieve the value associated with a key using square bracket notation. @@ -97,7 +96,6 @@ def __setitem__(self, key, value): # print("set", key, value) self.data[str(key)] = value - def __delitem__(self, key): """Delete the key-value pair associated with the specified key. @@ -190,7 +188,6 @@ def parameter(self, attribute, position=0): expand = Parameter.expand(value)[position] return expand - def boolean(self, key, value): """Set a boolean value based on string representation. diff --git a/src/cloudmesh/common/wifi.py b/src/cloudmesh/common/wifi.py index 996e5be..ec1edb4 100644 --- a/src/cloudmesh/common/wifi.py +++ b/src/cloudmesh/common/wifi.py @@ -20,7 +20,8 @@ class Wifi: location = "/etc/wpa_supplicant/wpa_supplicant.conf" - template_key = textwrap.dedent(""" + template_key = textwrap.dedent( + """ ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev update_config=1 country={country} @@ -29,9 +30,11 @@ class Wifi: ssid="{ssid}" key_mgmt=NONE }} - """) + """ + ) - template_psk = textwrap.dedent(""" + template_psk = textwrap.dedent( + """ ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev update_config=1 country=US @@ -41,17 +44,15 @@ class Wifi: psk="{password}" key_mgmt=WPA-PSK }} - """) # noqa: W293 + """ + ) # noqa: W293 template = template_psk @staticmethod - def set(ssid=None, - password=None, - country="US", - psk=True, - location=location, - sudo=False): + def set( + ssid=None, password=None, country="US", psk=True, location=location, sudo=False + ): """Sets the wifi. Only works for psk based wifi Args: From 17c03bdc0bd532f4c880e88b1c8aa3a04ff1de4e Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 09:32:41 -0500 Subject: [PATCH 02/16] cahnge back to the new use of tzlocal --- src/cloudmesh/common/DateTime.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/cloudmesh/common/DateTime.py b/src/cloudmesh/common/DateTime.py index 86ffe43..c46229b 100644 --- a/src/cloudmesh/common/DateTime.py +++ b/src/cloudmesh/common/DateTime.py @@ -39,7 +39,7 @@ class DateTime(object): @staticmethod def now(): return str(TIME.datetime.now()) - + @staticmethod def utc_now(): return str(TIME.datetime.utcnow()) @@ -87,20 +87,10 @@ def local(time): @staticmethod def utc_to_local(time): -<<<<<<< HEAD - TIME_FORMAT = "%Y-%m-%d %H:%M:%S" - utc = TIME.datetime.utcnow().strftime(TIME_FORMAT) - timestamp = calendar.timegm( - (TIME.datetime.strptime(utc, TIME_FORMAT)).timetuple() - ) - local = TIME.datetime.fromtimestamp(timestamp).strftime(TIME_FORMAT) - return local + " " + str(DateTime.timezone) -======= TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%f' utc = TIME.datetime.strptime(str(time), TIME_FORMAT) local_dt = utc.replace(tzinfo=ZoneInfo('UTC')).astimezone(tzlocal.get_localzone()) return local_dt.strftime(TIME_FORMAT) + " " + str(DateTime.timezone) ->>>>>>> 117453248a3f9136fa9720dddc7f0caba4bec3ba if __name__ == "__main__": From 88ac78f71794f1ec62e5fa485984f0daf264aeb6 Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 09:46:38 -0500 Subject: [PATCH 03/16] add date time tests --- tests/test_date.py | 106 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/test_date.py diff --git a/tests/test_date.py b/tests/test_date.py new file mode 100644 index 0000000..5d6c544 --- /dev/null +++ b/tests/test_date.py @@ -0,0 +1,106 @@ +############################################################### +# pytest -v --capture=no tests/test_date.py +# pytest -v --capture=no tests/test_date..py::test_date.test_001 +# pytest -v tests/test_date.py +############################################################### + +import pytest +from cloudmesh.common.util import HEADING +from cloudmesh.common.DateTime import DateTime + +class TestDateTime: + + def test_print(self): + HEADING() + + start = DateTime.now() + stop = DateTime.now() + DateTime.delta(1) + + print("START", start) + print("STOP", stop) + print("HUMANIZE STOP", DateTime.humanize(stop - start)) + print("LOCAL", DateTime.local(start)) + print("UTC", DateTime.utc(start)) + print("NATURAL", DateTime.natural(start)) + print("WORDS", DateTime.words(start)) + print("TIMEZONE", DateTime.timezone) + + # print("CONVERT", DateTime.local("2019-08-03 20:48:27.205442 UTC")) + """ + START 2019-08-03 21:34:14.019147 + STOP 2019-08-03 21:34:15.019150 + HUMANIZE STOP a second ago + LOCAL 2019-08-03 17:34:14 EST + UTC 2019-08-03 21:34:14.019147 UTC + NATURAL 2019-08-03 21:34:14.019147 UTC + WORDS Sat 6 Aug 2019, 21:34:14 UTC + TIMEZONE EST + """ + + + def test_now(self): + HEADING() + now = DateTime.now() + print(now) + assert isinstance(now, str) + + def test_utc_now(self): + HEADING() + utc_now = DateTime.utc_now() + print(utc_now) + assert isinstance(utc_now, str) + + def test_natural(self): + HEADING() + time = "2019-08-03 21:34:14" + natural_time = DateTime.natural(time) + print(natural_time) + assert isinstance(natural_time, str) + + def test_words(self): + HEADING() + time = "2019-08-03 21:34:14" + words = DateTime.words(time) + print(words) + assert isinstance(words, str) + + def test_datetime(self): + HEADING() + time = "2019-08-03 21:34:14" + datetime = DateTime.datetime(time) + print(datetime) + assert isinstance(datetime, str) + + def test_humanize(self): + HEADING() + time = "2019-08-03 21:34:14" + humanized_time = DateTime.humanize(time) + print(humanized_time) + assert isinstance(humanized_time, str) + + def test_string(self): + HEADING() + time = "2019-08-03 21:34:14" + string_time = DateTime.string(time) + print(string_time) + assert isinstance(string_time, str) + + def test_delta(self): + HEADING() + delta = DateTime.delta(1) + print(delta) + assert isinstance(delta, datetime.timedelta) + + def test_utc(self): + HEADING() + time = "2019-08-03 21:34:14" + utc_time = DateTime.utc(time) + print(utc_time) # Print utc_time before assert + assert isinstance(utc_time, str) + + def test_local(self): + HEADING() + time = "2019-08-03 21:34:14" + local_time = DateTime.local(time) + print(local_time) # Print local_time before assert + assert isinstance(local_time, str) \ No newline at end of file From 74311a0354bb8988e57b78b3c76df65c48a4b49d Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 09:53:06 -0500 Subject: [PATCH 04/16] update test for date --- tests/test_date.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_date.py b/tests/test_date.py index 5d6c544..bd41001 100644 --- a/tests/test_date.py +++ b/tests/test_date.py @@ -8,6 +8,16 @@ from cloudmesh.common.util import HEADING from cloudmesh.common.DateTime import DateTime +# ./cloudmesh-cc/tests/test_199_workflow_clean.py:from cloudmesh.common.DateTime import DateTime +# ./cloudmesh-cc/src/cloudmesh/cc/labelmaker.py:from cloudmesh.common.DateTime import DateTime +# ./cloudmesh-cc/src/cloudmesh/cc/workflow.py:from cloudmesh.common.DateTime import DateTime +# ./cloudmesh-multipass/src/cloudmesh/multipass/Provider.py:from cloudmesh.common.DateTime import DateTime +# ./cloudmesh-common/tests/test_date.py:from cloudmesh.common.DateTime import DateTime +# ./cloudmesh-common/src/cloudmesh/common/Host.py:from cloudmesh.common.DateTime import DateTime +# ./cloudmesh-common/src/cloudmesh/common/systeminfo.py:from cloudmesh.common.DateTime import DateTime +# ./cloudmesh-common/src/cloudmesh/common/StopWatch.py:from cloudmesh.common.DateTime import DateTime +# ./cloudmesh-common/src/cloudmesh/common/Printer.py:from cloudmesh.common.DateTime import DateTime + class TestDateTime: def test_print(self): From 1d7bc626db9d48a55acfbb3e9c8f69f997ebfb66 Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 10:07:14 -0500 Subject: [PATCH 05/16] sort failed datetime tests to the end --- src/cloudmesh/common/DateTime.py | 6 ++- tests/test_date.py | 93 ++++++++++++++++---------------- 2 files changed, 51 insertions(+), 48 deletions(-) diff --git a/src/cloudmesh/common/DateTime.py b/src/cloudmesh/common/DateTime.py index c46229b..8115ea5 100644 --- a/src/cloudmesh/common/DateTime.py +++ b/src/cloudmesh/common/DateTime.py @@ -87,9 +87,11 @@ def local(time): @staticmethod def utc_to_local(time): - TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%f' + TIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" utc = TIME.datetime.strptime(str(time), TIME_FORMAT) - local_dt = utc.replace(tzinfo=ZoneInfo('UTC')).astimezone(tzlocal.get_localzone()) + local_dt = utc.replace(tzinfo=ZoneInfo("UTC")).astimezone( + tzlocal.get_localzone() + ) return local_dt.strftime(TIME_FORMAT) + " " + str(DateTime.timezone) diff --git a/tests/test_date.py b/tests/test_date.py index bd41001..82640c0 100644 --- a/tests/test_date.py +++ b/tests/test_date.py @@ -18,36 +18,8 @@ # ./cloudmesh-common/src/cloudmesh/common/StopWatch.py:from cloudmesh.common.DateTime import DateTime # ./cloudmesh-common/src/cloudmesh/common/Printer.py:from cloudmesh.common.DateTime import DateTime -class TestDateTime: - - def test_print(self): - HEADING() - - start = DateTime.now() - stop = DateTime.now() + DateTime.delta(1) - - print("START", start) - print("STOP", stop) - print("HUMANIZE STOP", DateTime.humanize(stop - start)) - print("LOCAL", DateTime.local(start)) - print("UTC", DateTime.utc(start)) - print("NATURAL", DateTime.natural(start)) - print("WORDS", DateTime.words(start)) - print("TIMEZONE", DateTime.timezone) - - # print("CONVERT", DateTime.local("2019-08-03 20:48:27.205442 UTC")) - """ - START 2019-08-03 21:34:14.019147 - STOP 2019-08-03 21:34:15.019150 - HUMANIZE STOP a second ago - LOCAL 2019-08-03 17:34:14 EST - UTC 2019-08-03 21:34:14.019147 UTC - NATURAL 2019-08-03 21:34:14.019147 UTC - WORDS Sat 6 Aug 2019, 21:34:14 UTC - TIMEZONE EST - """ - +class TestDateTime: def test_now(self): HEADING() now = DateTime.now() @@ -67,13 +39,6 @@ def test_natural(self): print(natural_time) assert isinstance(natural_time, str) - def test_words(self): - HEADING() - time = "2019-08-03 21:34:14" - words = DateTime.words(time) - print(words) - assert isinstance(words, str) - def test_datetime(self): HEADING() time = "2019-08-03 21:34:14" @@ -88,6 +53,22 @@ def test_humanize(self): print(humanized_time) assert isinstance(humanized_time, str) + def test_utc(self): + HEADING() + time = "2019-08-03 21:34:14" + utc_time = DateTime.utc(time) + print(utc_time) # Print utc_time before assert + assert isinstance(utc_time, str) + + # class failed: + + def test_local(self): + HEADING() + time = "2019-08-03 21:34:14" + local_time = DateTime.local(time) + print(local_time) # Print local_time before assert + assert isinstance(local_time, str) + def test_string(self): HEADING() time = "2019-08-03 21:34:14" @@ -98,19 +79,39 @@ def test_string(self): def test_delta(self): HEADING() delta = DateTime.delta(1) - print(delta) + print(delta) assert isinstance(delta, datetime.timedelta) - def test_utc(self): + def test_words(self): HEADING() time = "2019-08-03 21:34:14" - utc_time = DateTime.utc(time) - print(utc_time) # Print utc_time before assert - assert isinstance(utc_time, str) + words = DateTime.words(time) + print(words) + assert isinstance(words, str) - def test_local(self): + def test_print(self): HEADING() - time = "2019-08-03 21:34:14" - local_time = DateTime.local(time) - print(local_time) # Print local_time before assert - assert isinstance(local_time, str) \ No newline at end of file + + start = DateTime.now() + stop = DateTime.now() + DateTime.delta(1) + + print("START", start) + print("STOP", stop) + print("HUMANIZE STOP", DateTime.humanize(stop - start)) + print("LOCAL", DateTime.local(start)) + print("UTC", DateTime.utc(start)) + print("NATURAL", DateTime.natural(start)) + print("WORDS", DateTime.words(start)) + print("TIMEZONE", DateTime.timezone) + + # print("CONVERT", DateTime.local("2019-08-03 20:48:27.205442 UTC")) + """ + START 2019-08-03 21:34:14.019147 + STOP 2019-08-03 21:34:15.019150 + HUMANIZE STOP a second ago + LOCAL 2019-08-03 17:34:14 EST + UTC 2019-08-03 21:34:14.019147 UTC + NATURAL 2019-08-03 21:34:14.019147 UTC + WORDS Sat 6 Aug 2019, 21:34:14 UTC + TIMEZONE EST + """ From 64a9f2c5579f9060b52a62ec53f8f2dba24dd92e Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 10:13:52 -0500 Subject: [PATCH 06/16] check if %f is in time format --- src/cloudmesh/common/DateTime.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cloudmesh/common/DateTime.py b/src/cloudmesh/common/DateTime.py index 8115ea5..1422852 100644 --- a/src/cloudmesh/common/DateTime.py +++ b/src/cloudmesh/common/DateTime.py @@ -87,7 +87,11 @@ def local(time): @staticmethod def utc_to_local(time): - TIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" + if "." in str(time): + TIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" + else: + TIME_FORMAT = "%Y-%m-%d %H:%M:%S" + utc = TIME.datetime.strptime(str(time), TIME_FORMAT) local_dt = utc.replace(tzinfo=ZoneInfo("UTC")).astimezone( tzlocal.get_localzone() From 89b715bbd6bd05a73bb2afa694ff7badd0715036 Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 11:21:31 -0500 Subject: [PATCH 07/16] remove deprecated utcnow --- src/cloudmesh/common/DateTime.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/cloudmesh/common/DateTime.py b/src/cloudmesh/common/DateTime.py index 1422852..1fd9b81 100644 --- a/src/cloudmesh/common/DateTime.py +++ b/src/cloudmesh/common/DateTime.py @@ -42,8 +42,8 @@ def now(): @staticmethod def utc_now(): - return str(TIME.datetime.utcnow()) - + return str(TIME.datetime.now(datetime.UTC)) + @staticmethod def natural(time): if type(time) == TIME.datetime: @@ -67,15 +67,20 @@ def humanize(time): @staticmethod def string(time): + # deprecated use get if type(time) == str: d = parser.parse(time) else: d = time return d + @staticmethod + def get(str): + return parser + @staticmethod def delta(n): - return TIME.timedelta(seconds=n) + return str(TIME.timedelta(seconds=n)) @staticmethod def utc(time): @@ -98,13 +103,21 @@ def utc_to_local(time): ) return local_dt.strftime(TIME_FORMAT) + " " + str(DateTime.timezone) + def datetime(str): + d = parser.parse(str) + @staticmethod + def add(one, two): + return DateTime.datetime(one) + DateTime.datetime(two) + if __name__ == "__main__": start = DateTime.now() - stop = DateTime.now() + DateTime.delta(1) + stop = DateTime.add(DateTime.now(), DateTime.delta(1)) + + stop2 = DateTime.datetime(DateTime.now()), DateTime.datetime(DateTime.delta(2)) print("START", start) - print("STOP", stop) + print("STOP", stop, stop2) print("HUMANIZE STOP", DateTime.humanize(stop - start)) print("LOCAL", DateTime.local(start)) print("UTC", DateTime.utc(start)) From 7178b64a104b56123cd38536403c5be8199fd65a Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 11:22:26 -0500 Subject: [PATCH 08/16] fix utcnow --- src/cloudmesh/common/DateTime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudmesh/common/DateTime.py b/src/cloudmesh/common/DateTime.py index 1fd9b81..a7ed65b 100644 --- a/src/cloudmesh/common/DateTime.py +++ b/src/cloudmesh/common/DateTime.py @@ -42,7 +42,7 @@ def now(): @staticmethod def utc_now(): - return str(TIME.datetime.now(datetime.UTC)) + return str(TIME.datetime.now(TIME.UTC)) @staticmethod def natural(time): From 1e634cc13f6badea612e734724d278e7e87c7ed1 Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 11:52:22 -0500 Subject: [PATCH 09/16] making DateTime test pass --- src/cloudmesh/common/DateTime.py | 45 ++++++++++++++++++++++---------- tests/test_date.py | 18 ++++++++----- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/cloudmesh/common/DateTime.py b/src/cloudmesh/common/DateTime.py index a7ed65b..cb452d1 100644 --- a/src/cloudmesh/common/DateTime.py +++ b/src/cloudmesh/common/DateTime.py @@ -43,7 +43,7 @@ def now(): @staticmethod def utc_now(): return str(TIME.datetime.now(TIME.UTC)) - + @staticmethod def natural(time): if type(time) == TIME.datetime: @@ -52,7 +52,11 @@ def natural(time): @staticmethod def words(time): - return TIME.datetime.strftime(time, "%a %w %b %Y, %H:%M:%S UTC") + if type(time) == TIME.datetime: + t = time + else: + t = DateTime.datetime(time) + return TIME.datetime.strftime(t, "%a %w %b %Y, %H:%M:%S UTC") @staticmethod def datetime(time): @@ -67,20 +71,31 @@ def humanize(time): @staticmethod def string(time): - # deprecated use get - if type(time) == str: - d = parser.parse(time) + if type(time) == TIME: + d = str(time) else: - d = time - return d + try: + d = parser.parse(time) + except: + d = DateTime.utc_to_local(time) + return str(d) @staticmethod - def get(str): - return parser + def get(time_str): + return parser.parse(time_str) @staticmethod def delta(n): - return str(TIME.timedelta(seconds=n)) + """ + Returns a timedelta object representing a time duration of `n` seconds. + + Parameters: + n (int): The number of seconds for the time duration. + + Returns: + timedelta: A timedelta object representing the time duration in datetime and not string format. + """ + return TIME.timedelta(seconds=n) @staticmethod def utc(time): @@ -103,13 +118,15 @@ def utc_to_local(time): ) return local_dt.strftime(TIME_FORMAT) + " " + str(DateTime.timezone) - def datetime(str): - d = parser.parse(str) + def datetime(time_str): + d = parser.parse(time_str) + return d @staticmethod def add(one, two): - return DateTime.datetime(one) + DateTime.datetime(two) - + return DateTime.datetime(DateTime.datetime(one)) + DateTime.datetime(two) + + if __name__ == "__main__": start = DateTime.now() stop = DateTime.add(DateTime.now(), DateTime.delta(1)) diff --git a/tests/test_date.py b/tests/test_date.py index 82640c0..db54d48 100644 --- a/tests/test_date.py +++ b/tests/test_date.py @@ -7,6 +7,8 @@ import pytest from cloudmesh.common.util import HEADING from cloudmesh.common.DateTime import DateTime +import datetime + # ./cloudmesh-cc/tests/test_199_workflow_clean.py:from cloudmesh.common.DateTime import DateTime # ./cloudmesh-cc/src/cloudmesh/cc/labelmaker.py:from cloudmesh.common.DateTime import DateTime @@ -42,9 +44,10 @@ def test_natural(self): def test_datetime(self): HEADING() time = "2019-08-03 21:34:14" - datetime = DateTime.datetime(time) - print(datetime) - assert isinstance(datetime, str) + d = DateTime.datetime(time) + print(d) + print(type(d)) + assert isinstance(d, datetime.datetime) def test_humanize(self): HEADING() @@ -80,7 +83,9 @@ def test_delta(self): HEADING() delta = DateTime.delta(1) print(delta) - assert isinstance(delta, datetime.timedelta) + print(type(delta)) + assert delta == datetime.timedelta(seconds=1) + assert str(delta) == "0:00:01" def test_words(self): HEADING() @@ -92,8 +97,9 @@ def test_words(self): def test_print(self): HEADING() - start = DateTime.now() - stop = DateTime.now() + DateTime.delta(1) + start = DateTime.datetime(DateTime.now()) + + stop = start + DateTime.delta(2) print("START", start) print("STOP", stop) From 37054a818dd2497b14ed6845781e560501f9438d Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 11:58:44 -0500 Subject: [PATCH 10/16] bump version 5.0.45 --- VERSION | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 1bd8b6c..53264bf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.44 +5.0.45 diff --git a/pyproject.toml b/pyproject.toml index 78791fb..0918a6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires = [ [project] name = "cloudmesh-common" -version = "5.0.44" +version = "5.0.45" description = "A set of useful APIs for cloudmesh" readme = "README.md" requires-python = ">=3.8" From cf97b64864c6fb8a32cd546ba39a1f834f6f2eab Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 11:59:06 -0500 Subject: [PATCH 11/16] bump version 5.0.46 --- VERSION | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 53264bf..52d8cda 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.45 +5.0.46 diff --git a/pyproject.toml b/pyproject.toml index 0918a6c..9f6e8d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires = [ [project] name = "cloudmesh-common" -version = "5.0.45" +version = "5.0.46" description = "A set of useful APIs for cloudmesh" readme = "README.md" requires-python = ">=3.8" From a899cac337885f2a8bc50ca29858326362f11ead Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 14:30:24 -0500 Subject: [PATCH 12/16] simplify readme generation that includes the manual page --- makefile.mk | 5 + src/cloudmesh/common/Shell.py | 626 +++++++++++++++-------------- src/cloudmesh/common/bin/readme.py | 78 ---- 3 files changed, 337 insertions(+), 372 deletions(-) delete mode 100644 src/cloudmesh/common/bin/readme.py diff --git a/makefile.mk b/makefile.mk index 5bfc79a..e2cd033 100644 --- a/makefile.mk +++ b/makefile.mk @@ -1,5 +1,6 @@ UNAME=$(shell uname) VERSION=`head -1 VERSION` +COMMAND := $(subst cloudmesh-,,$(package)) define banner @echo @@ -19,6 +20,10 @@ source: welcome ## Install the package in source mode pip: welcome ## Install the package in pip mode pip install -e . --config-settings editable_mode=strict + +readme: + cms man readme -p --command=${COMMAND} + ############################################################################## # CHECK ############################################################################## diff --git a/src/cloudmesh/common/Shell.py b/src/cloudmesh/common/Shell.py index 26016e8..2af51f4 100755 --- a/src/cloudmesh/common/Shell.py +++ b/src/cloudmesh/common/Shell.py @@ -53,10 +53,11 @@ # f(args) # return new_f + def windows_not_supported(f): """ - This is a decorator function that checks if the current platform is Windows. - If it is, it prints an error message and returns an empty string. + This is a decorator function that checks if the current platform is Windows. + If it is, it prints an error message and returns an empty string. If it's not, it simply calls the decorated function with the provided arguments. Args: @@ -65,6 +66,7 @@ def windows_not_supported(f): Returns: function: The decorated function which will either execute the original function or return an empty string based on the platform. """ + def wrapper(*args, **kwargs): host = get_platform() if host == "windows": @@ -90,6 +92,7 @@ def NotImplementedInWindows(): Console.error(f"The method {__name__} is not implemented in Windows.") sys.exit() + class Brew(object): """ This class provides methods to interact with the Homebrew package manager on macOS. @@ -182,40 +185,40 @@ def __init__(self, cmd, returncode, stderr, stdout): self.stdout = stdout def __str__(self): - """ - Returns a string representation of the Shell object. + """ + Returns a string representation of the Shell object. - The string includes the command, exit code, stderr, and stdout (if available). + The string includes the command, exit code, stderr, and stdout (if available). - Returns: - str: A string representation of the Shell object. - """ + Returns: + str: A string representation of the Shell object. + """ - def indent(lines, amount, ch=' '): - """indent the lines by multiples of ch + def indent(lines, amount, ch=" "): + """indent the lines by multiples of ch - Args: - lines - amount - ch + Args: + lines + amount + ch - Returns: + Returns: - """ - padding = amount * ch - return padding + ('\n' + padding).join(lines.split('\n')) + """ + padding = amount * ch + return padding + ("\n" + padding).join(lines.split("\n")) - cmd = ' '.join(map(quote, self.cmd)) - s = '' - s += 'Command: %s\n' % cmd - s += 'Exit code: %s\n' % self.returncode + cmd = " ".join(map(quote, self.cmd)) + s = "" + s += "Command: %s\n" % cmd + s += "Exit code: %s\n" % self.returncode - if self.stderr: - s += 'Stderr:\n' + indent(self.stderr, 4) - if self.stdout: - s += 'Stdout:\n' + indent(self.stdout, 4) + if self.stderr: + s += "Stderr:\n" + indent(self.stderr, 4) + if self.stdout: + s += "Stdout:\n" + indent(self.stdout, 4) - return s + return s class Subprocess(object): @@ -223,8 +226,9 @@ class Subprocess(object): instead you should use Shell. """ - def __init__(self, cmd, cwd=None, stderr=subprocess.PIPE, - stdout=subprocess.PIPE, env=None): + def __init__( + self, cmd, cwd=None, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=None + ): """execute the given command Args: @@ -234,10 +238,9 @@ def __init__(self, cmd, cwd=None, stderr=subprocess.PIPE, stdout: the pipe for the stdoutput env """ - Console.debug_msg('Running cmd: {}'.format(' '.join(map(quote, cmd)))) + Console.debug_msg("Running cmd: {}".format(" ".join(map(quote, cmd)))) - proc = subprocess.Popen(cmd, stderr=stderr, stdout=stdout, cwd=cwd, - env=env) + proc = subprocess.Popen(cmd, stderr=stderr, stdout=stdout, cwd=cwd, env=env) stdout, stderr = proc.communicate() self.returncode = proc.returncode @@ -245,8 +248,7 @@ def __init__(self, cmd, cwd=None, stderr=subprocess.PIPE, self.stdout = stdout if self.returncode != 0: - raise SubprocessError(cmd, self.returncode, self.stderr, - self.stdout) + raise SubprocessError(cmd, self.returncode, self.stderr, self.stdout) class Shell(object): @@ -255,6 +257,8 @@ class Shell(object): """ # Rest of the code... + + class Shell(object): """The shell class allowing us to conveniently access many operating system commands. TODO: This works well on Linux and OSX, but has not been tested much on Windows @@ -263,11 +267,7 @@ class Shell(object): # TODO: we have not supported cygwin for a while # cygwin_path = 'bin' # i copied fom C:\cygwin\bin - command = { - 'windows': {}, - 'linux': {}, - 'darwin': {} - } + command = {"windows": {}, "linux": {}, "darwin": {}} # TODO # @@ -302,7 +302,7 @@ def timezone(default="America/Indiana/Indianapolis"): Raises: - IndexError: If the system's timezone cannot be determined and no default timezone is provided. """ - + # BUG we need to be able to pass the default from the cmdline host = get_platform() if host == "windows": @@ -310,7 +310,9 @@ def timezone(default="America/Indiana/Indianapolis"): else: # result = Shell.run("ls -l /etc/localtime").strip().split("/") try: - result = Shell.run("ls -l /etc/localtime").strip().split("zoneinfo")[1][1:] + result = ( + Shell.run("ls -l /etc/localtime").strip().split("zoneinfo")[1][1:] + ) return result except IndexError as e: return default @@ -329,11 +331,13 @@ def locale(): """ try: - result = Shell.run('locale').split('\n')[0].split('_')[1].split('.')[0].lower() + result = ( + Shell.run("locale").split("\n")[0].split("_")[1].split(".")[0].lower() + ) return result except IndexError as e: Console.warning('Could not determine locale. Defaulting to "us"') - return 'us' + return "us" @staticmethod def ssh_enabled(): @@ -347,28 +351,30 @@ def ssh_enabled(): try: r = Shell.run("which sshd") except RuntimeError as e: - raise RuntimeError("You do not have OpenSSH installed. " - "sudo apt-get install openssh-client openssh-server " - "Automatic installation will be implemented in future cloudmesh version.") + raise RuntimeError( + "You do not have OpenSSH installed. " + "sudo apt-get install openssh-client openssh-server " + "Automatic installation will be implemented in future cloudmesh version." + ) # the reason why we do it like this is because WSL # does not have sshd as a status. this works fine r = Shell.run("service ssh status | fgrep running").strip() - + return len(r) > 0 elif os_is_windows(): # r = Shell.run("ps | grep -F ssh") # return "ssh" in r - processes = psutil.process_iter(attrs=['name']) + processes = psutil.process_iter(attrs=["name"]) # Filter the processes for 'ssh' - ssh_processes = [p.info for p in processes if 'ssh' in p.info['name']] + ssh_processes = [p.info for p in processes if "ssh" in p.info["name"]] return len(ssh_processes) > 0 elif os_is_mac(): r = Shell.run("ps -ef") if "sshd" in r: - print('IT WORKS!!!!!') + print("IT WORKS!!!!!") else: - print('it doenst work :(') + print("it doenst work :(") return "sshd" in r return False @@ -382,7 +388,7 @@ def run_timed(label, command, encoding=None, service=None): encoding: the encoding service: a prefix to the stopwatch label - Returns: + Returns: """ _label = str(label) @@ -393,7 +399,7 @@ def run_timed(label, command, encoding=None, service=None): return str(result) @staticmethod - def run(command, exitcode="", encoding='utf-8', replace=True, timeout=None): + def run(command, exitcode="", encoding="utf-8", replace=True, timeout=None): """executes the command and returns the output as string Args: @@ -415,23 +421,22 @@ def run(command, exitcode="", encoding='utf-8', replace=True, timeout=None): try: if timeout is not None: - r = subprocess.check_output(command, - stderr=subprocess.STDOUT, - shell=True, - timeout=timeout) + r = subprocess.check_output( + command, stderr=subprocess.STDOUT, shell=True, timeout=timeout + ) else: - r = subprocess.check_output(command, - stderr=subprocess.STDOUT, - shell=True) + r = subprocess.check_output( + command, stderr=subprocess.STDOUT, shell=True + ) except subprocess.CalledProcessError as e: raise RuntimeError(f"{e.returncode} {e.output.decode()}") - if encoding is None or encoding == 'utf-8': - return str(r, 'utf-8') + if encoding is None or encoding == "utf-8": + return str(r, "utf-8") else: return r @staticmethod - def run2(command, encoding='utf-8'): + def run2(command, encoding="utf-8"): """executes the command and returns the output as string. This command also allows execution of 32 bit commands. @@ -442,7 +447,7 @@ def run2(command, encoding='utf-8'): Returns: """ - if platform.lower() == 'win32': + if platform.lower() == "win32": import ctypes class disable_file_system_redirection: @@ -459,31 +464,25 @@ def __exit__(self, type, value, traceback): with disable_file_system_redirection(): command = f"{command}" - r = subprocess.check_output(command, - stderr=subprocess.STDOUT, - shell=True) - if encoding is None or encoding == 'utf-8': - return str(r, 'utf-8') + r = subprocess.check_output( + command, stderr=subprocess.STDOUT, shell=True + ) + if encoding is None or encoding == "utf-8": + return str(r, "utf-8") else: return r - elif platform.lower() == 'linux' or platform.lower() == 'darwin': + elif platform.lower() == "linux" or platform.lower() == "darwin": command = f"{command}" - r = subprocess.check_output(command, - stderr=subprocess.STDOUT, - shell=True) - if encoding is None or encoding == 'utf-8': - return str(r, 'utf-8') + r = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) + if encoding is None or encoding == "utf-8": + return str(r, "utf-8") else: return r @classmethod - def execute(cls, - cmd, - arguments="", - shell=False, - cwd=None, - traceflag=True, - witherror=True): + def execute( + cls, cmd, arguments="", shell=False, cwd=None, traceflag=True, witherror=True + ): """Run Shell command Args: @@ -505,9 +504,9 @@ def execute(cls, terminal = cls.terminal_type() # print cls.command os_command = [cmd] - if terminal in ['linux', 'windows']: + if terminal in ["linux", "windows"]: os_command = [cmd] - elif 'cygwin' in terminal: + elif "cygwin" in terminal: if not cls.command_exists(cmd): print("ERROR: the command could not be found", cmd) return @@ -528,7 +527,7 @@ def execute(cls, # noinspection PyPep8 try: if shell: - if platform.lower() == 'win32': + if platform.lower() == "win32": import ctypes class disable_file_system_redirection: @@ -537,36 +536,32 @@ class disable_file_system_redirection: def __enter__(self): self.old_value = ctypes.c_long() - self.success = self._disable( - ctypes.byref(self.old_value)) + self.success = self._disable(ctypes.byref(self.old_value)) def __exit__(self, type, value, traceback): if self.success: self._revert(self.old_value) if len(os_command) == 1: - os_command = os_command[0].split(' ') + os_command = os_command[0].split(" ") with disable_file_system_redirection(): - result = subprocess.check_output(os_command, - stderr=subprocess.STDOUT, - shell=True, - cwd=cwd) + result = subprocess.check_output( + os_command, stderr=subprocess.STDOUT, shell=True, cwd=cwd + ) else: result = subprocess.check_output( - os_command, - stderr=subprocess.STDOUT, - shell=True, - cwd=cwd) + os_command, stderr=subprocess.STDOUT, shell=True, cwd=cwd + ) else: result = subprocess.check_output( os_command, # shell=True, stderr=subprocess.STDOUT, - cwd=cwd) + cwd=cwd, + ) except: # noqa: E722 if witherror: - Console.error("problem executing subprocess", - traceflag=traceflag) + Console.error("problem executing subprocess", traceflag=traceflag) if result is not None: result = result.strip().decode() return result @@ -592,7 +587,7 @@ def is_choco_installed(): if not os_is_windows(): return False try: - r = Shell.run('choco --version') + r = Shell.run("choco --version") # no problem return True except subprocess.CalledProcessError: @@ -608,9 +603,8 @@ def install_chocolatey(): import pyuac if os_is_windows(): - try: - r = Shell.run('choco --version') + r = Shell.run("choco --version") Console.ok("Chocolatey already installed") return True except subprocess.CalledProcessError: @@ -618,7 +612,7 @@ def install_chocolatey(): if not pyuac.isUserAdmin(): Console.error("Please run the terminal as administrator.") return False - + # Get the full path of the current Python script current_script_path = os.path.abspath(__file__) @@ -626,35 +620,40 @@ def install_chocolatey(): script_directory = os.path.dirname(current_script_path) # Join the script directory with "bin" - bin_directory = os.path.join(script_directory, 'bin') - print(f'Looking in {bin_directory} for install script...') + bin_directory = os.path.join(script_directory, "bin") + print(f"Looking in {bin_directory} for install script...") # Command to install Chocolatey using the Command Prompt - chocolatey_install_command = fr'powershell Start-Process -Wait -FilePath {bin_directory}\win-setup.bat' + chocolatey_install_command = rf"powershell Start-Process -Wait -FilePath {bin_directory}\win-setup.bat" print(chocolatey_install_command) # Run the Chocolatey installation command using subprocess and capture output - completed_process = subprocess.run(chocolatey_install_command, - shell=True, text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if 'current directory is invalid' in str(completed_process): - Console.error("You are currently standing in a non-existent directory.") + completed_process = subprocess.run( + chocolatey_install_command, + shell=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if "current directory is invalid" in str(completed_process): + Console.error( + "You are currently standing in a non-existent directory." + ) return False print(completed_process) Console.ok("Chocolatey installed") return True # try: - # process = subprocess.run('choco --version') - # Console.ok("Chocolatey installed") - # return True + # process = subprocess.run('choco --version') + # Console.ok("Chocolatey installed") + # return True # except subprocess.CalledProcessError: - # Console.error("Chocolatey was not added to path. Close and reopen terminal and execute previous command again.") - # return True - # pass + # Console.error("Chocolatey was not added to path. Close and reopen terminal and execute previous command again.") + # return True + # pass # except FileNotFoundError: - # Console.error("Chocolatey was not added to path. Close and reopen terminal and execute previous command again.") - # return True - # pass + # Console.error("Chocolatey was not added to path. Close and reopen terminal and execute previous command again.") + # return True + # pass else: Console.error("chocolatey can only be installed in Windows") return False @@ -674,14 +673,14 @@ def install_choco_package(package: str): Console.error("Chocolatey not installed, or terminal needs to be reloaded.") return False - command = f'cmd.exe /K choco install {package} -y' + command = f"cmd.exe /K choco install {package} -y" process = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True # Allows working with text output + universal_newlines=True, # Allows working with text output ) # Read and display the live output @@ -691,7 +690,6 @@ def install_choco_package(package: str): # Wait for the subprocess to complete process.wait() return True - @staticmethod def install_brew(): @@ -705,59 +703,57 @@ def install_brew(): # elevate() - if not os_is_mac(): Console.error("Homebrew can only be installed on mac") return False - + try: - r = subprocess.check_output("brew --version", - stderr=subprocess.STDOUT, - shell=True) + r = subprocess.check_output( + "brew --version", stderr=subprocess.STDOUT, shell=True + ) Console.ok("Homebrew already installed") return True except subprocess.CalledProcessError: Console.info("Installing Homebrew...") - content = """#!/bin/bash pw="$(osascript -e 'Tell application "System Events" to display dialog "Please enter your macOS password to install Homebrew:" default answer "" with hidden answer' -e 'text returned of result' 2>/dev/null)" && echo "$pw" """ - askpass = os.path.expanduser('~/pw.sh') + askpass = os.path.expanduser("~/pw.sh") if not os.path.isfile(askpass): writefile(askpass, content) - os.system('chmod +x ~/pw.sh') - + os.system("chmod +x ~/pw.sh") + # command = 'NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"' command = f'osascript -e \'tell application "Terminal" to do script "/bin/bash -c \\"export SUDO_ASKPASS={askpass} ; export NONINTERACTIVE=1 ; $(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\\""\'' print(command) # try: subprocess.run(command, shell=True, check=True) - # print("Homebrew installed successfully.") + # print("Homebrew installed successfully.") # except subprocess.CalledProcessError as e: - # print(f"Error while installing Homebrew: {e}") + # print(f"Error while installing Homebrew: {e}") while True: from time import sleep + try: - r = subprocess.check_output("brew --version", - stderr=subprocess.STDOUT, - shell=True) + r = subprocess.check_output( + "brew --version", stderr=subprocess.STDOUT, shell=True + ) Console.ok("Homebrew installed") sleep(8) return True except subprocess.CalledProcessError: # print('Waiting', end=' ') - + sleep(2) continue # Console.error("Homebrew could not be installed.") # return False - + Shell.rm(askpass) - @staticmethod def is_root(): @@ -792,7 +788,7 @@ def rmdir(top, verbose=False): # assert False, f"Directory {top} could not be deleted" @staticmethod - def dot2svg(filename, engine='dot'): + def dot2svg(filename, engine="dot"): """ Converts a Graphviz DOT file to SVG format using the specified engine. @@ -803,10 +799,7 @@ def dot2svg(filename, engine='dot'): Returns: None """ - data = { - 'engine': engine, - 'file': filename.replace(".dot", "") - } + data = {"engine": engine, "file": filename.replace(".dot", "")} command = "{engine} -Tsvg {file}.dot > {file}.svg".format(**data) print(command) os.system(command) @@ -844,10 +837,9 @@ def map_filename(name): if result.host == "localhost": _name = "." - if _name.startswith("http"): result.path = _name - result.protocol = _name.split(':', 1)[0] + result.protocol = _name.split(":", 1)[0] elif _name.startswith("wsl:"): result.path = _name.replace("wsl:", "") # Abbreviations: replace ~ with home dir and ./ + / with pwd @@ -855,7 +847,9 @@ def map_filename(name): if os_is_linux(): result.path = result.path.replace("~", f"/mnt/c/home/{result.user}") else: - result.path = result.path.replace("~", f"/mnt/c/Users/{result.user}") + result.path = result.path.replace( + "~", f"/mnt/c/Users/{result.user}" + ) elif not result.path.startswith("/"): if os_is_windows(): pwd = pwd.replace("C:", "/mnt/c").replace("\\", "/") @@ -913,7 +907,7 @@ def browser(filename=None): Returns: """ - if not os.path.isabs(filename) and 'http' not in filename: + if not os.path.isabs(filename) and "http" not in filename: filename = Shell.map_filename(filename).path webbrowser.open(filename, new=2, autoraise=False) @@ -921,11 +915,11 @@ def browser(filename=None): def fetch(filename=None, destination=None): """ Fetches the content of a file specified by the filename and returns it. - + Parameters: - filename (str): The path or URL of the file to fetch. - destination (str): The path where the fetched content should be saved (optional). - + Returns: - str: The content of the fetched file. """ @@ -933,7 +927,7 @@ def fetch(filename=None, destination=None): content = None - if _filename.path.startswith('http'): + if _filename.path.startswith("http"): result = requests.get(_filename.path) content = result.text @@ -957,10 +951,10 @@ def terminal_title(name): Returns: void: This function does not return any value. """ - return f'echo -n -e \"\033]0;{name}\007\"' + return f'echo -n -e "\033]0;{name}\007"' @classmethod - def terminal(cls, command='pwd', title=None, kind=None): + def terminal(cls, command="pwd", title=None, kind=None): """starts a terminal and executes the command in that terminal Args: @@ -974,16 +968,16 @@ def terminal(cls, command='pwd', title=None, kind=None): """ # title nameing not implemented print(platform) - if platform == 'darwin': + if platform == "darwin": label = Shell.terminal_title(title) os.system( - f"osascript -e 'tell application \"Terminal\" to do script \"{command}\"'" + f'osascript -e \'tell application "Terminal" to do script "{command}"\'' ) elif platform == "linux": # for ubuntu running gnome dist = os_platform.linux_distribution()[0] - linux_apps = {'ubuntu': 'gnome-terminal', 'debian': 'lxterminal'} - os.system(f"{linux_apps[dist]} -e \"bash -c \'{command}; exec $SHELL\'\"") + linux_apps = {"ubuntu": "gnome-terminal", "debian": "lxterminal"} + os.system(f"{linux_apps[dist]} -e \"bash -c '{command}; exec $SHELL'\"") elif platform == "win32": if kind is None: @@ -991,17 +985,19 @@ def terminal(cls, command='pwd', title=None, kind=None): kind = "gitbash" kind = kind.lower() if kind == "gitbash": - p = subprocess.Popen([r"C:\Program Files\Git\git-bash.exe", - "-c", - f"{command}"]) + p = subprocess.Popen( + [r"C:\Program Files\Git\git-bash.exe", "-c", f"{command}"] + ) return p.pid elif kind == "cmd": Console.error(f"Command not implemented for {kind}") elif kind == "powershell": Console.error(f"Command not implemented for {kind}") else: - Console.error("Git bash is not found, please make sure it " - "is installed. Other terminals not supported.") + Console.error( + "Git bash is not found, please make sure it " + "is installed. Other terminals not supported." + ) raise NotImplementedError else: raise NotImplementedError @@ -1020,22 +1016,20 @@ def live(cls, command, cwd=None): """ if cwd is None: cwd = os.getcwd() - process = subprocess.Popen(shlex.split(command), cwd=cwd, - stdout=subprocess.PIPE) - result = b'' + process = subprocess.Popen( + shlex.split(command), cwd=cwd, stdout=subprocess.PIPE + ) + result = b"" while True: output = process.stdout.read(1) - if output == b'' and process.poll() is not None: + if output == b"" and process.poll() is not None: break if output: result = result + output sys.stdout.write(output.decode("utf-8")) sys.stdout.flush() rc = process.poll() - data = dotdict({ - "status": rc, - "text": output.decode("utf-8") - }) + data = dotdict({"status": rc, "text": output.decode("utf-8")}) return data @staticmethod @@ -1056,7 +1050,7 @@ def get_python(cls): """ python_version = sys.version_info[:3] v_string = [str(i) for i in python_version] - python_version_s = '.'.join(v_string) + python_version_s = ".".join(v_string) # pip_version = pip.__version__ pip_version = Shell.pip("--version").split()[1] @@ -1081,7 +1075,6 @@ def check_output(cls, *args, **kwargs): """ return subprocess.check_output(*args, **kwargs) - @classmethod def ls(cls, directory=".", match=None): """ @@ -1095,6 +1088,7 @@ def ls(cls, directory=".", match=None): list: A list of filenames matching the given pattern (if provided) in the specified directory. """ import re + if match == None: files = os.listdir(directory) else: @@ -1116,20 +1110,19 @@ def gpu_name(cls): pass return name - @classmethod # @NotImplementedInWindows def unix_ls(cls, *args): - """ - Executes the linux 'ls' command with the given arguments. + """ + Executes the linux 'ls' command with the given arguments. - Args: - *args: Additional arguments to be passed to the 'ls' command. + Args: + *args: Additional arguments to be passed to the 'ls' command. - Returns: - The output of the 'ls' command as a string. - """ - return cls.execute('ls', args) + Returns: + The output of the 'ls' command as a string. + """ + return cls.execute("ls", args) @staticmethod def ps(short=False, attributes=None): @@ -1151,9 +1144,20 @@ def ps(short=False, attributes=None): if attributes is not None: pinfo = proc.as_dict(attrs=attributes) elif short: - pinfo = proc.as_dict(attrs=['pid', 'name', 'cmdline', 'ppid', 'username', - 'status', 'create_time', 'terminal', 'cwd', - 'open_files']) + pinfo = proc.as_dict( + attrs=[ + "pid", + "name", + "cmdline", + "ppid", + "username", + "status", + "create_time", + "terminal", + "cwd", + "open_files", + ] + ) else: pinfo = proc.as_dict() except psutil.NoSuchProcess: @@ -1176,7 +1180,7 @@ def bash(cls, *args): Returns: The output of the bash command. """ - return cls.execute('bash', args) + return cls.execute("bash", args) @classmethod # @NotImplementedInWindows @@ -1190,7 +1194,7 @@ def brew(cls, *args): The output of the 'brew' command. """ NotImplementedInWindows() - return cls.execute('brew', args) + return cls.execute("brew", args) @classmethod # @NotImplementedInWindows @@ -1207,7 +1211,7 @@ def cat(cls, *args): content = readfile(args[0]) return content else: - return cls.execute('cat', args) + return cls.execute("cat", args) @classmethod def git(cls, *args): @@ -1219,7 +1223,7 @@ def git(cls, *args): Returns: The output of the git command execution. """ - return cls.execute('git', args) + return cls.execute("git", args) # noinspection PyPep8Naming @classmethod @@ -1236,7 +1240,7 @@ def VBoxManage(cls, *args): if platform == "darwin": command = "/Applications/VirtualBox.app/Contents/MacOS/VBoxManage" else: - command = 'VBoxManage' + command = "VBoxManage" return cls.execute(command, args) @classmethod @@ -1249,7 +1253,7 @@ def blockdiag(cls, *args): Returns: The result of executing blockdiag with the given arguments. """ - return cls.execute('blockdiag', args) + return cls.execute("blockdiag", args) @classmethod def cm(cls, *args): @@ -1261,11 +1265,10 @@ def cm(cls, *args): Returns: None """ - return cls.execute('cm', args) - + return cls.execute("cm", args) @staticmethod - def cms(command, encoding='utf-8'): + def cms(command, encoding="utf-8"): """ Executes a command using the 'cms' command-line tool. @@ -1278,7 +1281,7 @@ def cms(command, encoding='utf-8'): """ return Shell.run("cms " + command, encoding=encoding) - + @classmethod def cms_exec(cls, *args): """executes cms with the given arguments @@ -1289,7 +1292,7 @@ def cms_exec(cls, *args): Returns: The output of the 'cms' command execution """ - return cls.execute('cms', args) + return cls.execute("cms", args) @classmethod def cmsd(cls, *args): @@ -1301,7 +1304,7 @@ def cmsd(cls, *args): Returns: None """ - return cls.execute('cmsd', args) + return cls.execute("cmsd", args) @classmethod def head(cls, filename=None, lines=10): @@ -1315,7 +1318,7 @@ def head(cls, filename=None, lines=10): str: The output of the head command. """ filename = cls.map_filename(filename).path - r = Shell.run(f'head -n {lines} {filename}') + r = Shell.run(f"head -n {lines} {filename}") return r @classmethod @@ -1328,7 +1331,7 @@ def keystone(cls, *args): Returns: The output of the keystone command. """ - return cls.execute('keystone', args) + return cls.execute("keystone", args) @staticmethod def kill_pid(pid): @@ -1344,7 +1347,7 @@ def kill_pid(pid): Raises: subprocess.CalledProcessError: If the process is already down. """ - if sys.platform == 'win32': + if sys.platform == "win32": try: result = Shell.run(f"taskkill /IM {pid} /F") except Exception as e: @@ -1353,7 +1356,7 @@ def kill_pid(pid): try: result = Shell.kill("-9", pid) except subprocess.CalledProcessError: - result = 'server is already down...' + result = "server is already down..." return result @classmethod @@ -1369,7 +1372,7 @@ def kill(cls, *args): """ NotImplementedInWindows() # TODO: use tasklisk, compare to linux - return cls.execute('kill', args) + return cls.execute("kill", args) @classmethod def download(cls, source, destination, force=False, provider=None, chunk_size=128): @@ -1400,25 +1403,31 @@ def download(cls, source, destination, force=False, provider=None, chunk_size=12 if os.path.exists(destination) and not force: return destination - if provider == 'system': + if provider == "system": # First try wget - wget_return = os.system(f'wget -O {destination} {source}') + wget_return = os.system(f"wget -O {destination} {source}") if wget_return == 0: - Console.info('Used wget') + Console.info("Used wget") return destination # Then curl - curl_return = os.system(f'curl -L -o {destination} {source}') + curl_return = os.system(f"curl -L -o {destination} {source}") if curl_return == 0: - Console.info('Used curl') + Console.info("Used curl") return destination # Default is requests lib. If provider is None, or if provider == 'system' # but wget and curl fail, default here r = requests.get(source, stream=True, allow_redirects=True) - total_size = int(r.headers.get('content-length')) - - with open(destination, 'wb') as fd: - with tqdm(total=total_size, unit="B", - unit_scale=True, desc=destination, initial=0, ascii=True) as pbar: + total_size = int(r.headers.get("content-length")) + + with open(destination, "wb") as fd: + with tqdm( + total=total_size, + unit="B", + unit_scale=True, + desc=destination, + initial=0, + ascii=True, + ) as pbar: for chunk in r.iter_content(chunk_size=chunk_size): fd.write(chunk) pbar.update(len(chunk)) @@ -1435,7 +1444,7 @@ def mount(cls, *args): Returns: None """ - return cls.execute('mount', args) + return cls.execute("mount", args) @classmethod def umount(cls, *args): @@ -1447,7 +1456,7 @@ def umount(cls, *args): Returns: None """ - return cls.execute('umount', args) + return cls.execute("umount", args) @classmethod def nova(cls, *args): @@ -1459,7 +1468,7 @@ def nova(cls, *args): Returns: The output of the 'nova' command execution. """ - return cls.execute('nova', args) + return cls.execute("nova", args) @classmethod def ping(cls, host=None, count=1): @@ -1473,11 +1482,11 @@ def ping(cls, host=None, count=1): str: the output of the ping command """ r = None - option = '-n' if os_is_windows() else '-c' - parameters = "{option} {count} {host}".format(option=option, - count=count, - host=host) - r = Shell.run(f'ping {parameters}') + option = "-n" if os_is_windows() else "-c" + parameters = "{option} {count} {host}".format( + option=option, count=count, host=host + ) + r = Shell.run(f"ping {parameters}") if r is None: Console.error("ping is not installed") return r @@ -1504,7 +1513,7 @@ def rackdiag(cls, *args): Returns: The result of executing rackdiag with the given arguments. """ - return cls.execute('rackdiag', args) + return cls.execute("rackdiag", args) @staticmethod def count_files(directory, recursive=False): @@ -1557,7 +1566,7 @@ def rsync(cls, *args): Returns: None """ - return cls.execute('rsync', args) + return cls.execute("rsync", args) @classmethod def scp(cls, *args): @@ -1569,7 +1578,7 @@ def scp(cls, *args): Returns: None """ - return cls.execute('scp', args) + return cls.execute("scp", args) @classmethod # @NotImplementedInWindows @@ -1587,7 +1596,7 @@ def sort(cls, *args): """ NotImplementedInWindows() # TODO: https://superuser.com/questions/1316317/is-there-a-windows-equivalent-to-the-unix-uniq - return cls.execute('sort', args) + return cls.execute("sort", args) @classmethod def sh(cls, *args): @@ -1599,7 +1608,7 @@ def sh(cls, *args): Returns: The output of the executed command as a string. """ - return cls.execute('sh', args) + return cls.execute("sh", args) @classmethod def ssh(cls, *args): @@ -1611,7 +1620,7 @@ def ssh(cls, *args): Returns: The output of the ssh command execution """ - return cls.execute('ssh', args) + return cls.execute("ssh", args) @classmethod # @NotImplementedInWindows @@ -1627,7 +1636,7 @@ def sudo(cls, *args): """ NotImplementedInWindows() # TODO: https://stackoverflow.com/questions/9652720/how-to-run-sudo-command-in-windows - return cls.execute('sudo', args) + return cls.execute("sudo", args) @classmethod def tail(cls, filename=None, lines=10): @@ -1641,7 +1650,7 @@ def tail(cls, filename=None, lines=10): str: The output of the tail command. """ filename = cls.map_filename(filename).path - r = Shell.run(f'tail -n {lines} {filename}') + r = Shell.run(f"tail -n {lines} {filename}") return r @classmethod @@ -1654,7 +1663,7 @@ def vagrant(cls, *args): Returns: The output of the vagrant command execution """ - return cls.execute('vagrant', args, shell=True) + return cls.execute("vagrant", args, shell=True) @classmethod def pandoc(cls, *args): @@ -1666,7 +1675,7 @@ def pandoc(cls, *args): Returns: The output of the pandoc command. """ - return cls.execute('pandoc', args) + return cls.execute("pandoc", args) @classmethod def mongod(cls, *args): @@ -1678,7 +1687,7 @@ def mongod(cls, *args): Returns: The output of the mongod command execution """ - return cls.execute('mongod', args) + return cls.execute("mongod", args) @classmethod # @NotImplementedInWindows @@ -1692,7 +1701,7 @@ def dialog(cls, *args): The output of the dialog command. """ NotImplementedInWindows() - return cls.execute('dialog', args) + return cls.execute("dialog", args) @classmethod def pip(cls, *args): @@ -1704,7 +1713,7 @@ def pip(cls, *args): Returns: The output of the pip command execution """ - return cls.execute('pip', args) + return cls.execute("pip", args) @classmethod def fgrep(cls, string=None, file=None): @@ -1721,9 +1730,9 @@ def fgrep(cls, string=None, file=None): """ if not os_is_windows(): - r = Shell.run(f'fgrep {string} {file}') + r = Shell.run(f"fgrep {string} {file}") else: - r = Shell.run(f'grep -F {string} {file}') + r = Shell.run(f"grep -F {string} {file}") return r @classmethod @@ -1742,7 +1751,7 @@ def grep(cls, string=None, file=None): >>> Shell.grep('pattern', 'file.txt') 'line containing pattern' """ - r = Shell.run(f'grep {string} {file}') + r = Shell.run(f"grep {string} {file}") return r @classmethod @@ -1831,6 +1840,32 @@ def find_lines_from(cls, lines, what): result = result + [line] return result + @staticmethod + def replace_lines_between(lines, what_from, what_to, content): + """Replaces the content between the markers with the specified content. + + Args: + lines (str): The multiline string to search within. + what_from (str): The starting marker. + what_to (str): The ending marker. + content (str): The content to replace the lines between the markers. + + Returns: + str: The modified multiline string. + """ + lines = lines.splitlines() + start_index = None + end_index = None + for i, line in enumerate(lines): + if what_from in line: + start_index = i + if what_to in line: + end_index = i + break + if start_index is not None and end_index is not None: + lines[start_index + 1 : end_index] = content.splitlines() + return "\n".join(lines) + @classmethod def find_lines_between(cls, lines, what_from, what_to): """returns all lines that come between the markers @@ -1880,15 +1915,15 @@ def terminal_type(cls): """ what = sys.platform - kind = 'UNDEFINED_TERMINAL_TYPE' - if 'linux' in what: - kind = 'linux' - elif 'darwin' in what: - kind = 'darwin' - elif 'cygwin' in what: - kind = 'cygwin' - elif what in ['windows', 'win32']: - kind = 'windows' + kind = "UNDEFINED_TERMINAL_TYPE" + if "linux" in what: + kind = "linux" + elif "darwin" in what: + kind = "darwin" + elif "cygwin" in what: + kind = "cygwin" + elif what in ["windows", "win32"]: + kind = "windows" return kind @@ -1946,7 +1981,6 @@ def get_pid(name, service="psutil"): pid = proc.pid return pid - @classmethod def check_python(cls): """checks if the python version is supported @@ -1958,13 +1992,14 @@ def check_python(cls): v_string = [str(i) for i in python_version] - python_version_s = '.'.join(v_string) + python_version_s = ".".join(v_string) if python_version[0] == 2: - print( "You are running an unsupported version of python: {:}".format( - python_version_s)) + python_version_s + ) + ) # python_version_s = '.'.join(v_string) # if (python_version[0] == 2) and (python_version[1] >= 7) and (python_version[2] >= 9): @@ -1975,27 +2010,31 @@ def check_python(cls): # print(" We recommend you update your python") elif python_version[0] == 3: - - if (python_version[0] == 3) and (python_version[1] >= 7) and (python_version[2] >= 0): - + if ( + (python_version[0] == 3) + and (python_version[1] >= 7) + and (python_version[2] >= 0) + ): print( "You are running a supported version of python: {:}".format( - python_version_s)) + python_version_s + ) + ) else: print( "WARNING: You are running an unsupported version of python: {:}".format( - python_version_s)) + python_version_s + ) + ) print(" We recommend you update your python") # pip_version = pip.__version__ python_version, pip_version = cls.get_python() if int(pip_version.split(".")[0]) >= 19: - print("You are running a supported version of pip: " + str( - pip_version)) + print("You are running a supported version of pip: " + str(pip_version)) else: - print("WARNING: You are running an old version of pip: " + str( - pip_version)) + print("WARNING: You are running an old version of pip: " + str(pip_version)) print(" We recommend you update your pip with \n") print(" pip install -U pip\n") @@ -2018,7 +2057,9 @@ def copy_source(cls, source, destination): if os.path.isfile(source): # If the source is a file shutil.copy2(source, destination) elif os.path.isdir(source): # If the source is a directory - shutil.copytree(source, os.path.join(destination, os.path.basename(source))) + shutil.copytree( + source, os.path.join(destination, os.path.basename(source)) + ) else: Console.error(f"'{source}' is neither a file nor a directory.") except Exception as e: @@ -2065,17 +2106,19 @@ def copy_file(cls, source, destination, verbose=False): print(" source :", s.path) print(" destination:", d.path) - if s.protocol in ['http', 'https']: - command = f'curl {s.path} -o {d.path}' + if s.protocol in ["http", "https"]: + command = f"curl {s.path} -o {d.path}" Shell.run(command) - elif s.protocol == 'scp': - Shell.run(f'scp {s.user}@{s.host}:{s.path} {d.path}') - elif s.protocol == 'ftp': - command = fr"curl -u {s.username}:{s.password} ftp://{s.ftp_path} -o {d.path}" + elif s.protocol == "scp": + Shell.run(f"scp {s.user}@{s.host}:{s.path} {d.path}") + elif s.protocol == "ftp": + command = ( + rf"curl -u {s.username}:{s.password} ftp://{s.ftp_path} -o {d.path}" + ) print(command) Shell.run(command) - elif d.protocol == 'ftp': - command = fr"curl -T {s.path} ftp://{d.ftp_path} --user {d.username}:{d.password}" + elif d.protocol == "ftp": + command = rf"curl -T {s.path} ftp://{d.ftp_path} --user {d.username}:{d.password}" print(command) Shell.run(command) else: @@ -2112,7 +2155,6 @@ def mkdir(cls, directory): Console.error(e, traceflag=True) return False - def unzip(cls, source_filename, dest_dir): """Unzips a file into the destination directory. @@ -2127,12 +2169,12 @@ def unzip(cls, source_filename, dest_dir): for member in zf.infolist(): # Path traversal defense copied from # http://hg.python.org/cpython/file/tip/Lib/http/server.py#l789 - words = member.filename.split('/') + words = member.filename.split("/") path = path_expand(dest_dir) for word in words[:-1]: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) - if word in (os.curdir, os.pardir, ''): + if word in (os.curdir, os.pardir, ""): continue path = os.path.join(path, word) zf.extract(member, path) @@ -2156,7 +2198,7 @@ def try_program(program): return True def run_edit_program(program, file): - cmd = f''' + cmd = f""" osascript < **Note:** The README.md page is automatically generated, do not edit it. - > To modify, change the content in - > - > Curly brackets must use two in README-source.md. - """ - - # Find icons - icons = """ - [![image](https://img.shields.io/pypi/v/{repo}.svg)](https://pypi.org/project/{repo}/) - [![Python](https://img.shields.io/pypi/pyversions/{repo}.svg)](https://pypi.python.org/pypi/{repo}) - [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/cloudmesh/{repo}/blob/main/LICENSE) - [![Format](https://img.shields.io/pypi/format/{repo}.svg)](https://pypi.python.org/pypi/{repo}) - [![Status](https://img.shields.io/pypi/status/{repo}.svg)](https://pypi.python.org/pypi/{repo}) - [![Travis](https://travis-ci.com/cloudmesh/{repo}.svg?branch=main)](https://travis-ci.com/cloudmesh/{repo}) - """ - - # Find Tests - tests = sorted(glob.glob('tests/test_**.py')) - links = [ - "[{name}]({x})".format(x=x, name=os.path.basename(x).replace('.py', '')) for - x in tests] - - tests = " * " + "\n * ".join(links) - - # Get manual - if repo == "cloudmesh-installer": - manual = Shell.run("cloudmesh-installer --help") - else: - manual = Shell.run(f"cms help {command}") - - man = [] - start = False - for line in manual.splitlines(): - start = start or "Usage:" in line - if start: - if not line.startswith("Timer:"): - man.append(line) - manual = textwrap.dedent('\n'.join(man)).strip() - manual = "```bash\n" + manual + "\n```\n" - - # Create README - source = readfile("README-source.md") - readme = source.format(**locals()) - writefile("README.md", readme) - -# Command-line execution -if __name__ == "__main__": - repo = sys.argv[1] if len(sys.argv) > 1 else "" - command = sys.argv[2] if len(sys.argv) > 2 else "" - generate_readme(repo, command) From 2f7386adc0cedf1298044773c3cd5572c485d461 Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 16:12:18 -0500 Subject: [PATCH 13/16] improve README.md --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index b988459..094c5e4 100644 --- a/README.md +++ b/README.md @@ -131,13 +131,9 @@ from cloudmesh.common.StopWatch import StopWatch import time StopWatch.start("a") - time.sleep(3) - StopWatch.stop("a") - StopWatch.status("a", True) - StopWatch.benchmark() ``` From c81a1de568dfca49c63306baebdebd2a3e91d1be Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 23:28:46 -0500 Subject: [PATCH 14/16] fix toml --- pyproject.toml | 2 +- src/cloudmesh/common/ssh/ssh_config.py | 40 +++++++++++++++----------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9f6e8d4..d7e4126 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,4 +75,4 @@ Changelog = "https://github.com/cloudmesh/cloudmesh-common/blob/main/CHANGELOG.m [tool.setuptools.packages.find] where = ["src"] -include = ["cloudmesh.common"] +include = ["cloudmesh.common", "cloudmesh.common.*"] diff --git a/src/cloudmesh/common/ssh/ssh_config.py b/src/cloudmesh/common/ssh/ssh_config.py index 8c92d61..ed2eb3c 100644 --- a/src/cloudmesh/common/ssh/ssh_config.py +++ b/src/cloudmesh/common/ssh/ssh_config.py @@ -11,7 +11,7 @@ # noinspection PyPep8Naming -class ssh_config(object): +class ssh_config: """Managing the config in .ssh""" def __init__(self, filename=None): @@ -39,7 +39,7 @@ def names(self): attribute, value = line.split(" ", 1) attribute = attribute.strip() value = value.strip() - if attribute.lower() in ['host']: + if attribute.lower() in ["host"]: found_names.append(value) return found_names @@ -51,20 +51,22 @@ def load(self): return self.hosts with open(self.filename) as f: content = f.readlines() - content = [" ".join(x.split()).strip('\n').lstrip().split(' ', 1) for x in content] + content = [ + " ".join(x.split()).strip("\n").lstrip().split(" ", 1) for x in content + ] # removes duplicated spaces, and splits in two fields, removes leading spaces hosts = {} host = "NA" for line in content: - if line[0].startswith('#') or line[0] == '': + if line[0].startswith("#") or line[0] == "": pass # ignore line else: attribute = line[0] value = line[1] - if attribute.lower().strip() in ['Host', 'host']: + if attribute.lower().strip() in ["Host", "host"]: host = value - hosts[host] = {'host': host} + hosts[host] = {"host": host} else: # In case of special configuration lines, such as port # forwarding, @@ -119,9 +121,9 @@ def execute(self, name, command): """ if name in ["localhost"]: - r = '\n'.join(Shell.sh("-c", command).split()[-1:]) + r = "\n".join(Shell.sh("-c", command).split()[-1:]) else: - r = '\n'.join(Shell.ssh(name, command).split()[-1:]) + r = "\n".join(Shell.ssh(name, command).split()[-1:]) return r def local(self, command): @@ -166,13 +168,15 @@ def delete(name): writefile(filename, result) - def generate(self, - host="india", - hostname="india.futuresystems.org", - identity="~/.ssh/id_rsa.pub", - user=None, - force=False, - verbose=False): + def generate( + self, + host="india", + hostname="india.futuresystems.org", + identity="~/.ssh/id_rsa.pub", + user=None, + force=False, + verbose=False, + ): """adds a host to the config file with given parameters. #TODO: make sure this is better documented Args: @@ -190,12 +194,14 @@ def generate(self, Console.error(f"{host} already in ~/.ssh/config", traceflag=False) return "" else: - entry = dedent(f""" + entry = dedent( + f""" Host {host} Hostname {hostname} User {user} IdentityFile {identity} - """) + """ + ) try: with open(self.filename, "a") as config_ssh: config_ssh.write(entry) From 8c052fda2df58dbd945b2e0773e97ed9505aa6f9 Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 23:30:22 -0500 Subject: [PATCH 15/16] bump version 5.0.47 --- VERSION | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 52d8cda..c2bea52 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.46 +5.0.47 diff --git a/pyproject.toml b/pyproject.toml index d7e4126..f71cfb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires = [ [project] name = "cloudmesh-common" -version = "5.0.46" +version = "5.0.47" description = "A set of useful APIs for cloudmesh" readme = "README.md" requires-python = ">=3.8" From b0e44932bc6c6c71917b4bd5bbe8937163b74a3a Mon Sep 17 00:00:00 2001 From: Gregor von Laszewski Date: Tue, 26 Dec 2023 23:31:07 -0500 Subject: [PATCH 16/16] bump version 5.0.48 --- VERSION | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index c2bea52..be59a03 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.47 +5.0.48 diff --git a/pyproject.toml b/pyproject.toml index f71cfb6..8e0727a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires = [ [project] name = "cloudmesh-common" -version = "5.0.47" +version = "5.0.48" description = "A set of useful APIs for cloudmesh" readme = "README.md" requires-python = ">=3.8"