From 36083852d7031e4464fa6778ac44b1222369a5e2 Mon Sep 17 00:00:00 2001 From: Lee Kelvin Date: Thu, 6 Jun 2024 05:32:53 -0700 Subject: [PATCH 1/4] Move pipeline-dot build from cmdLineFwk to builder This commit moves the pipeline-dot build logic from cmdLineFwk into cli/script/build.py. As part of this commit, a switch to using the same back-end display args parser as --show pipeline-graph is also made. --- python/lsst/ctrl/mpexec/cli/cmd/commands.py | 6 ++++-- python/lsst/ctrl/mpexec/cli/script/build.py | 10 ++++++++++ python/lsst/ctrl/mpexec/cmdLineFwk.py | 5 +---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/python/lsst/ctrl/mpexec/cli/cmd/commands.py b/python/lsst/ctrl/mpexec/cli/cmd/commands.py index 70204f2c..69f327f8 100644 --- a/python/lsst/ctrl/mpexec/cli/cmd/commands.py +++ b/python/lsst/ctrl/mpexec/cli/cmd/commands.py @@ -124,10 +124,12 @@ def build(ctx: click.Context, **kwargs: Any) -> None: """ kwargs = _collectActions(ctx, **kwargs) show = ShowInfo(kwargs.pop("show", [])) - if kwargs.get("butler_config") is not None and {"pipeline-graph", "task-graph"}.isdisjoint(show.commands): + if kwargs.get("butler_config") is not None and ( + {"pipeline-graph", "task-graph"}.isdisjoint(show.commands) and not kwargs.get("pipeline_dot") + ): raise click.ClickException( "--butler-config was provided but nothing uses it " - "(only --show pipeline-graph and --show task-graph do)." + "(only --show pipeline-graph, --show task-graph and --pipeline-dot do)." ) script.build(**kwargs, show=show) _unhandledShow(show, "build") diff --git a/python/lsst/ctrl/mpexec/cli/script/build.py b/python/lsst/ctrl/mpexec/cli/script/build.py index 40de4f21..9628b3a7 100644 --- a/python/lsst/ctrl/mpexec/cli/script/build.py +++ b/python/lsst/ctrl/mpexec/cli/script/build.py @@ -28,6 +28,7 @@ from types import SimpleNamespace from lsst.daf.butler import Butler +from lsst.pipe.base.pipeline_graph import visualization from ... import CmdLineFwk from ..utils import _PipelineAction @@ -114,6 +115,15 @@ def build( # type: ignore else: butler = None + if pipeline_dot: + with open(pipeline_dot, "w") as stream: + visualization.show_dot( + pipeline.to_graph(butler.registry if butler is not None else None), + stream, + dataset_types=True, + task_classes="full", + ) + show.show_pipeline_info(pipeline, butler=butler) return pipeline diff --git a/python/lsst/ctrl/mpexec/cmdLineFwk.py b/python/lsst/ctrl/mpexec/cmdLineFwk.py index ff742ec7..e5b959f4 100644 --- a/python/lsst/ctrl/mpexec/cmdLineFwk.py +++ b/python/lsst/ctrl/mpexec/cmdLineFwk.py @@ -75,7 +75,7 @@ from lsst.utils.logging import getLogger from lsst.utils.threads import disable_implicit_threading -from .dotTools import graph2dot, pipeline2dot +from .dotTools import graph2dot from .executionGraphFixup import ExecutionGraphFixup from .mpGraphExecutor import MPGraphExecutor from .preExecInit import PreExecInit, PreExecInitLimited @@ -582,9 +582,6 @@ def makePipeline(self, args: SimpleNamespace) -> Pipeline: if args.save_pipeline: pipeline.write_to_uri(args.save_pipeline) - if args.pipeline_dot: - pipeline2dot(pipeline, args.pipeline_dot) - return pipeline def makeGraph(self, pipeline: Pipeline, args: SimpleNamespace) -> QuantumGraph | None: From 3f16742e77624bd416269a052c1861aaea6a24df Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Thu, 6 Jun 2024 16:39:16 -0400 Subject: [PATCH 2/4] Use new visualization-only resolution when displaying PipelineGraphs. --- python/lsst/ctrl/mpexec/cli/script/build.py | 2 +- python/lsst/ctrl/mpexec/showInfo.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/lsst/ctrl/mpexec/cli/script/build.py b/python/lsst/ctrl/mpexec/cli/script/build.py index 9628b3a7..7b00d333 100644 --- a/python/lsst/ctrl/mpexec/cli/script/build.py +++ b/python/lsst/ctrl/mpexec/cli/script/build.py @@ -118,7 +118,7 @@ def build( # type: ignore if pipeline_dot: with open(pipeline_dot, "w") as stream: visualization.show_dot( - pipeline.to_graph(butler.registry if butler is not None else None), + pipeline.to_graph(butler.registry if butler is not None else None, visualization_only=True), stream, dataset_types=True, task_classes="full", diff --git a/python/lsst/ctrl/mpexec/showInfo.py b/python/lsst/ctrl/mpexec/showInfo.py index ce9c4cfb..c4c38b8a 100644 --- a/python/lsst/ctrl/mpexec/showInfo.py +++ b/python/lsst/ctrl/mpexec/showInfo.py @@ -172,9 +172,13 @@ def show_pipeline_info(self, pipeline: Pipeline, butler: Butler | None) -> None: case "tasks": self._showTaskHierarchy(pipeline) case "pipeline-graph": - visualization.show(pipeline.to_graph(registry), self.stream, dataset_types=True) + visualization.show( + pipeline.to_graph(registry, visualization_only=True), self.stream, dataset_types=True + ) case "task-graph": - visualization.show(pipeline.to_graph(registry), self.stream, dataset_types=False) + visualization.show( + pipeline.to_graph(registry, visualization_only=True), self.stream, dataset_types=False + ) case _: raise RuntimeError(f"Unexpectedly tried to process command {command!r}.") self.handled.add(command) From 4c5ce04e1c412a3d568171d323c70858eb4899b8 Mon Sep 17 00:00:00 2001 From: Lee Kelvin Date: Thu, 20 Jun 2024 12:28:47 -0700 Subject: [PATCH 3/4] Modify unit tests for new show-pipeline outputs --- tests/test_cmdLineFwk.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_cmdLineFwk.py b/tests/test_cmdLineFwk.py index 78327652..3cf42066 100644 --- a/tests/test_cmdLineFwk.py +++ b/tests/test_cmdLineFwk.py @@ -478,11 +478,11 @@ def testShowPipeline(self): self.assertEqual( "\n".join( [ - "○ add_dataset_in", + "○ add_dataset_in: {detector} NumpyArray", "│", - "■ task", + "■ task: {detector}", "│", - "◍ add_dataset_out, add2_dataset_out", + "◍ add_dataset_out, add2_dataset_out: {detector} NumpyArray", ] ), output, @@ -493,7 +493,7 @@ def testShowPipeline(self): show.show_pipeline_info(pipeline, None) stream.seek(0) output = stream.read().strip() - self.assertEqual("■ task", output) + self.assertEqual("■ task: {detector}", output) stream = StringIO() show = ShowInfo(["config=task::addEnd"], stream=stream) # Match but warns From 28d8a12febe762b443b00b5404dacd8df60c58b0 Mon Sep 17 00:00:00 2001 From: Lee Kelvin Date: Sun, 23 Jun 2024 11:52:10 -0700 Subject: [PATCH 4/4] Add user-visible change document to doc/changes --- doc/changes/DM-44647.feature.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/changes/DM-44647.feature.md diff --git a/doc/changes/DM-44647.feature.md b/doc/changes/DM-44647.feature.md new file mode 100644 index 00000000..bee9c34a --- /dev/null +++ b/doc/changes/DM-44647.feature.md @@ -0,0 +1,6 @@ +Move pipeline-dot build from cmdLineFwk to builder. + +This PR moves the pipeline-dot build from the cmdLineFwk package to the builder package. +This is done to make the pipeline-dot build more accessible to other packages. +As part of this change, output pipeline-dot files contain dimensions and storage classes for each dataset. +This change also includes updates to existing unit tests to reflect the new output types.