From cedd11a41a1a66af4a4e8f5fa410dcda2134154d Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Tue, 7 Nov 2023 13:26:34 -0500 Subject: [PATCH] Python black linting Signed-off-by: Rob Syme --- bin/pipeline-gantt.py | 76 +++++++++++++------ .../templates/dumpsoftwareversions.py | 5 +- 2 files changed, 57 insertions(+), 24 deletions(-) diff --git a/bin/pipeline-gantt.py b/bin/pipeline-gantt.py index 8f0a29a..e8cd2cb 100755 --- a/bin/pipeline-gantt.py +++ b/bin/pipeline-gantt.py @@ -15,46 +15,76 @@ def extract_instance(fusion_logs: Path) -> str: for line_number, line in enumerate(file, start=1): try: log = json.loads(line) - if 'instance-id' in log: - return log['instance-id'] + if "instance-id" in log: + return log["instance-id"] except json.JSONDecodeError: print(f"WARN: invalid JSON at '{fusion_logs}' line {line_number}") return "" + @click.command() -@click.option('--title', default='Pipeline GANTT', help='Plot title.') -@click.option('--input-dir', type=click.Path(), help='The pipeline dump tar.gz input file.') -@click.option('--output-file', type=click.Path(), help='The HTML output file') +@click.option("--title", default="Pipeline GANTT", help="Plot title.") +@click.option( + "--input-dir", type=click.Path(), help="The pipeline dump tar.gz input file." +) +@click.option("--output-file", type=click.Path(), help="The HTML output file") def build_gantt(title: str, input_dir: str, output_file: str): tasks = [] instance_ids = {} - for path in Path(input_dir).glob('workflow-tasks.json'): + for path in Path(input_dir).glob("workflow-tasks.json"): with path.open() as json_file: tasks = json.load(json_file) - for path in Path(input_dir).glob('**/.fusion.log'): + for path in Path(input_dir).glob("**/.fusion.log"): task_id = int(path.parent.name) instance_id = extract_instance(path) instance_ids[task_id] = instance_id for t in tasks: - t['instanceId'] = instance_ids.get(t['taskId'], "unknow") - - data = [{k: v for k, v in t.items() if k in ['taskId', 'name', 'start', 'complete', 'memory', 'cpus', 'machineType', 'instanceId']} for t in tasks] - df = pd.DataFrame({ - 'id': f"T{d['taskId']}", - 'name': d['name'], - 'size': f"{d['cpus']}c_{d['memory'] / 1024 ** 3:.0f}GB", - 'start': datetime.strptime(d['start'], '%Y-%m-%dT%H:%M:%SZ'), - 'complete': datetime.strptime(d['complete'], '%Y-%m-%dT%H:%M:%SZ'), - 'instance': f"{d['instanceId']} ({d['machineType']})" - } - for d in data - ) - - fig = px.timeline(df, title=title, x_start="start", x_end="complete", y="id", color="instance", text="name", pattern_shape="size") + t["instanceId"] = instance_ids.get(t["taskId"], "unknow") + + data = [ + { + k: v + for k, v in t.items() + if k + in [ + "taskId", + "name", + "start", + "complete", + "memory", + "cpus", + "machineType", + "instanceId", + ] + } + for t in tasks + ] + df = pd.DataFrame( + { + "id": f"T{d['taskId']}", + "name": d["name"], + "size": f"{d['cpus']}c_{d['memory'] / 1024 ** 3:.0f}GB", + "start": datetime.strptime(d["start"], "%Y-%m-%dT%H:%M:%SZ"), + "complete": datetime.strptime(d["complete"], "%Y-%m-%dT%H:%M:%SZ"), + "instance": f"{d['instanceId']} ({d['machineType']})", + } + for d in data + ) + + fig = px.timeline( + df, + title=title, + x_start="start", + x_end="complete", + y="id", + color="instance", + text="name", + pattern_shape="size", + ) fig.write_html(output_file) -if __name__ == '__main__': +if __name__ == "__main__": build_gantt() diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py index cecaa10..c1e49ec 100755 --- a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -8,6 +8,7 @@ import platform from textwrap import dedent + def main(): """Load all version files and generate merged output.""" versions_this_module = {} @@ -17,7 +18,9 @@ def main(): } with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module + versions_by_process = ( + yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module + ) # aggregate versions by the module name (derived from fully-qualified process name) versions_by_module = {}