Skip to content

Commit

Permalink
Python black linting
Browse files Browse the repository at this point in the history
Signed-off-by: Rob Syme <[email protected]>
  • Loading branch information
robsyme committed Nov 7, 2023
1 parent 51248d8 commit cedd11a
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 24 deletions.
76 changes: 53 additions & 23 deletions bin/pipeline-gantt.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,46 +15,76 @@ def extract_instance(fusion_logs: Path) -> str:
for line_number, line in enumerate(file, start=1):
try:
log = json.loads(line)
if 'instance-id' in log:
return log['instance-id']
if "instance-id" in log:
return log["instance-id"]
except json.JSONDecodeError:
print(f"WARN: invalid JSON at '{fusion_logs}' line {line_number}")
return ""


@click.command()
@click.option('--title', default='Pipeline GANTT', help='Plot title.')
@click.option('--input-dir', type=click.Path(), help='The pipeline dump tar.gz input file.')
@click.option('--output-file', type=click.Path(), help='The HTML output file')
@click.option("--title", default="Pipeline GANTT", help="Plot title.")
@click.option(
"--input-dir", type=click.Path(), help="The pipeline dump tar.gz input file."
)
@click.option("--output-file", type=click.Path(), help="The HTML output file")
def build_gantt(title: str, input_dir: str, output_file: str):
tasks = []
instance_ids = {}

for path in Path(input_dir).glob('workflow-tasks.json'):
for path in Path(input_dir).glob("workflow-tasks.json"):
with path.open() as json_file:
tasks = json.load(json_file)
for path in Path(input_dir).glob('**/.fusion.log'):
for path in Path(input_dir).glob("**/.fusion.log"):
task_id = int(path.parent.name)
instance_id = extract_instance(path)
instance_ids[task_id] = instance_id

for t in tasks:
t['instanceId'] = instance_ids.get(t['taskId'], "unknow")

data = [{k: v for k, v in t.items() if k in ['taskId', 'name', 'start', 'complete', 'memory', 'cpus', 'machineType', 'instanceId']} for t in tasks]
df = pd.DataFrame({
'id': f"T{d['taskId']}",
'name': d['name'],
'size': f"{d['cpus']}c_{d['memory'] / 1024 ** 3:.0f}GB",
'start': datetime.strptime(d['start'], '%Y-%m-%dT%H:%M:%SZ'),
'complete': datetime.strptime(d['complete'], '%Y-%m-%dT%H:%M:%SZ'),
'instance': f"{d['instanceId']} ({d['machineType']})"
}
for d in data
)

fig = px.timeline(df, title=title, x_start="start", x_end="complete", y="id", color="instance", text="name", pattern_shape="size")
t["instanceId"] = instance_ids.get(t["taskId"], "unknow")

data = [
{
k: v
for k, v in t.items()
if k
in [
"taskId",
"name",
"start",
"complete",
"memory",
"cpus",
"machineType",
"instanceId",
]
}
for t in tasks
]
df = pd.DataFrame(
{
"id": f"T{d['taskId']}",
"name": d["name"],
"size": f"{d['cpus']}c_{d['memory'] / 1024 ** 3:.0f}GB",
"start": datetime.strptime(d["start"], "%Y-%m-%dT%H:%M:%SZ"),
"complete": datetime.strptime(d["complete"], "%Y-%m-%dT%H:%M:%SZ"),
"instance": f"{d['instanceId']} ({d['machineType']})",
}
for d in data
)

fig = px.timeline(
df,
title=title,
x_start="start",
x_end="complete",
y="id",
color="instance",
text="name",
pattern_shape="size",
)
fig.write_html(output_file)


if __name__ == '__main__':
if __name__ == "__main__":
build_gantt()

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit cedd11a

Please sign in to comment.