From 34fff729a5aeee3f4fc2fa100377fc0bb95e73c6 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Fri, 4 Aug 2023 13:18:11 -0400 Subject: [PATCH 1/6] Use match/case instead of if/elif in ShowInfo dispatch. --- python/lsst/ctrl/mpexec/showInfo.py | 52 +++++++++++++++-------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/python/lsst/ctrl/mpexec/showInfo.py b/python/lsst/ctrl/mpexec/showInfo.py index 9a0da16f..e8dd476c 100644 --- a/python/lsst/ctrl/mpexec/showInfo.py +++ b/python/lsst/ctrl/mpexec/showInfo.py @@ -141,21 +141,22 @@ def show_pipeline_info(self, pipeline: Pipeline) -> None: continue args = self.commands[command] - if command == "pipeline": - print(pipeline, file=self.stream) - elif command == "config": - for arg in args: - self._showConfig(pipeline, arg, False) - elif command == "dump-config": - for arg in args: - self._showConfig(pipeline, arg, True) - elif command == "history": - for arg in args: - self._showConfigHistory(pipeline, arg) - elif command == "tasks": - self._showTaskHierarchy(pipeline) - else: - raise RuntimeError(f"Unexpectedly tried to process command {command!r}.") + match command: + case "pipeline": + print(pipeline, file=self.stream) + case "config": + for arg in args: + self._showConfig(pipeline, arg, False) + case "dump-config": + for arg in args: + self._showConfig(pipeline, arg, True) + case "history": + for arg in args: + self._showConfigHistory(pipeline, arg) + case "tasks": + self._showTaskHierarchy(pipeline) + case _: + raise RuntimeError(f"Unexpectedly tried to process command {command!r}.") self.handled.add(command) def show_graph_info(self, graph: QuantumGraph, args: SimpleNamespace | None = None) -> None: @@ -172,16 +173,17 @@ def show_graph_info(self, graph: QuantumGraph, args: SimpleNamespace | None = No for command in self.graph_commands: if command not in self.commands: continue - if command == "graph": - self._showGraph(graph) - elif command == "uri": - if args is None: - raise ValueError("The uri option requires additional command line arguments.") - self._showUri(graph, args) - elif command == "workflow": - self._showWorkflow(graph) - else: - raise RuntimeError(f"Unexpectedly tried to process command {command!r}.") + match command: + case "graph": + self._showGraph(graph) + case "uri": + if args is None: + raise ValueError("The uri option requires additional command line arguments.") + self._showUri(graph, args) + case "workflow": + self._showWorkflow(graph) + case _: + raise RuntimeError(f"Unexpectedly tried to process command {command!r}.") self.handled.add(command) def _showConfig(self, pipeline: Pipeline, showArgs: str, dumpFullConfig: bool) -> None: From f9a956ff524d301d4fb1d79a175ff08ee2981985 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Fri, 4 Aug 2023 14:09:23 -0400 Subject: [PATCH 2/6] Add support for --show pipeline-graph and --show task-graph. --- python/lsst/ctrl/mpexec/cli/opt/options.py | 4 +++- python/lsst/ctrl/mpexec/showInfo.py | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/python/lsst/ctrl/mpexec/cli/opt/options.py b/python/lsst/ctrl/mpexec/cli/opt/options.py index 7dbcc866..40271d57 100644 --- a/python/lsst/ctrl/mpexec/cli/opt/options.py +++ b/python/lsst/ctrl/mpexec/cli/opt/options.py @@ -313,7 +313,9 @@ composition; ``graph`` to show information about quanta; ``workflow`` to show information about quanta and their dependency; ``tasks`` to show task composition; ``uri`` to show - predicted dataset URIs of quanta.""" + predicted dataset URIs of quanta; ``pipeline-graph`` for a + text-based visualization of the pipeline (tasks and dataset types); + ``task-graph`` for a text-based visualization of just the tasks.""" ), metavar="ITEM|ITEM=VALUE", multiple=True, diff --git a/python/lsst/ctrl/mpexec/showInfo.py b/python/lsst/ctrl/mpexec/showInfo.py index e8dd476c..1f695a48 100644 --- a/python/lsst/ctrl/mpexec/showInfo.py +++ b/python/lsst/ctrl/mpexec/showInfo.py @@ -41,6 +41,7 @@ import lsst.pex.config.history as pexConfigHistory from lsst.daf.butler import DatasetRef, DatasetType, DatastoreRecordData, NamedKeyMapping from lsst.pipe.base import Pipeline, QuantumGraph +from lsst.pipe.base.pipeline_graph import visualization from . import util from .cmdLineFwk import _ButlerFactory @@ -101,7 +102,15 @@ class ShowInfo: Raised if some show commands are not recognized. """ - pipeline_commands = {"pipeline", "config", "history", "tasks", "dump-config"} + pipeline_commands = { + "pipeline", + "config", + "history", + "tasks", + "dump-config", + "pipeline-graph", + "task-graph", + } graph_commands = {"graph", "workflow", "uri"} def __init__(self, show: list[str], stream: Any = None) -> None: @@ -155,6 +164,10 @@ def show_pipeline_info(self, pipeline: Pipeline) -> None: self._showConfigHistory(pipeline, arg) case "tasks": self._showTaskHierarchy(pipeline) + case "pipeline-graph": + visualization.show(pipeline.to_graph(), self.stream, dataset_types=True) + case "task-graph": + visualization.show(pipeline.to_graph(), self.stream, dataset_types=False) case _: raise RuntimeError(f"Unexpectedly tried to process command {command!r}.") self.handled.add(command) From 2a6ddbee2f6834050059fe26f6a953fc73aef0ef Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Fri, 4 Aug 2023 14:22:21 -0400 Subject: [PATCH 3/6] Add -b option to build to allow new --show options to do more. --- .../lsst/ctrl/mpexec/cli/opt/optionGroups.py | 1 + python/lsst/ctrl/mpexec/cli/opt/options.py | 4 ++- python/lsst/ctrl/mpexec/cli/script/build.py | 27 ++++++++++++++++--- python/lsst/ctrl/mpexec/showInfo.py | 12 ++++++--- tests/test_cmdLineFwk.py | 16 +++++------ 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py b/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py index 4fcd9acf..c2a4b2b8 100644 --- a/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py +++ b/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py @@ -75,6 +75,7 @@ def __init__(self) -> None: ctrlMpExecOpts.save_pipeline_option(), ctrlMpExecOpts.pipeline_dot_option(), pipeBaseOpts.instrument_option(help=instrumentOptionHelp, metavar="instrument", multiple=True), + ctrlMpExecOpts.butler_config_option(required=False), ] diff --git a/python/lsst/ctrl/mpexec/cli/opt/options.py b/python/lsst/ctrl/mpexec/cli/opt/options.py index 40271d57..05c200da 100644 --- a/python/lsst/ctrl/mpexec/cli/opt/options.py +++ b/python/lsst/ctrl/mpexec/cli/opt/options.py @@ -315,7 +315,9 @@ dependency; ``tasks`` to show task composition; ``uri`` to show predicted dataset URIs of quanta; ``pipeline-graph`` for a text-based visualization of the pipeline (tasks and dataset types); - ``task-graph`` for a text-based visualization of just the tasks.""" + ``task-graph`` for a text-based visualization of just the tasks. + With -b, pipeline-graph and task-graph include additional information. + """ ), metavar="ITEM|ITEM=VALUE", multiple=True, diff --git a/python/lsst/ctrl/mpexec/cli/script/build.py b/python/lsst/ctrl/mpexec/cli/script/build.py index 6afb2701..ae82af2d 100644 --- a/python/lsst/ctrl/mpexec/cli/script/build.py +++ b/python/lsst/ctrl/mpexec/cli/script/build.py @@ -27,12 +27,21 @@ from types import SimpleNamespace +from lsst.daf.butler import Butler + from ... import CmdLineFwk from ..utils import _PipelineAction def build( # type: ignore - order_pipeline, pipeline, pipeline_actions, pipeline_dot, save_pipeline, show, **kwargs + order_pipeline, + pipeline, + pipeline_actions, + pipeline_dot, + save_pipeline, + show, + butler_config=None, + **kwargs, ): """Implement the command line interface `pipetask build` subcommand. @@ -59,7 +68,14 @@ def build( # type: ignore Path location for storing resulting pipeline definition in YAML format. show : `lsst.ctrl.mpexec.showInfo.ShowInfo` Descriptions of what to dump to stdout. - kwargs : `dict` [`str`, `str`] + butler_config : `str`, `dict`, or `lsst.daf.butler.Config`, optional + If `str`, `butler_config` is the path location of the gen3 + butler/registry config file. If `dict`, `butler_config` is key value + pairs used to init or update the `lsst.daf.butler.Config` instance. If + `Config`, it is the object used to configure a Butler. + Only used to resolve pipeline graphs for --show pipeline-graph and + --show task-graph. + **kwargs Ignored; click commands may accept options for more than one script function and pass all the option kwargs to each of the script functions which ingore these unused kwargs. @@ -93,6 +109,11 @@ def build( # type: ignore # Will raise an exception if it fails to build the pipeline. pipeline = f.makePipeline(args) - show.show_pipeline_info(pipeline) + if butler_config is not None: + butler = Butler(butler_config, writeable=False) + else: + butler = None + + show.show_pipeline_info(pipeline, butler=butler) return pipeline diff --git a/python/lsst/ctrl/mpexec/showInfo.py b/python/lsst/ctrl/mpexec/showInfo.py index 1f695a48..b7d7287a 100644 --- a/python/lsst/ctrl/mpexec/showInfo.py +++ b/python/lsst/ctrl/mpexec/showInfo.py @@ -39,7 +39,7 @@ import lsst.pex.config as pexConfig import lsst.pex.config.history as pexConfigHistory -from lsst.daf.butler import DatasetRef, DatasetType, DatastoreRecordData, NamedKeyMapping +from lsst.daf.butler import Butler, DatasetRef, DatasetType, DatastoreRecordData, NamedKeyMapping from lsst.pipe.base import Pipeline, QuantumGraph from lsst.pipe.base.pipeline_graph import visualization @@ -137,7 +137,7 @@ def unhandled(self) -> frozenset[str]: """Return the commands that have not yet been processed.""" return frozenset(set(self.commands) - self.handled) - def show_pipeline_info(self, pipeline: Pipeline) -> None: + def show_pipeline_info(self, pipeline: Pipeline, butler: Butler | None) -> None: """Display useful information about the pipeline. Parameters @@ -145,6 +145,10 @@ def show_pipeline_info(self, pipeline: Pipeline) -> None: pipeline : `lsst.pipe.base.Pipeline` The pipeline to use when reporting information. """ + if butler is not None: + registry = butler.registry + else: + registry = None for command in self.pipeline_commands: if command not in self.commands: continue @@ -165,9 +169,9 @@ def show_pipeline_info(self, pipeline: Pipeline) -> None: case "tasks": self._showTaskHierarchy(pipeline) case "pipeline-graph": - visualization.show(pipeline.to_graph(), self.stream, dataset_types=True) + visualization.show(pipeline.to_graph(registry), self.stream, dataset_types=True) case "task-graph": - visualization.show(pipeline.to_graph(), self.stream, dataset_types=False) + visualization.show(pipeline.to_graph(registry), self.stream, dataset_types=False) case _: raise RuntimeError(f"Unexpectedly tried to process command {command!r}.") self.handled.add(command) diff --git a/tests/test_cmdLineFwk.py b/tests/test_cmdLineFwk.py index e7cd13e4..4c9a35f4 100644 --- a/tests/test_cmdLineFwk.py +++ b/tests/test_cmdLineFwk.py @@ -429,7 +429,7 @@ def testShowPipeline(self): ["pipeline", "config", "history=task::addend", "tasks", "dump-config", "config=task::add*"], stream=stream, ) - show.show_pipeline_info(pipeline) + show.show_pipeline_info(pipeline, None) self.assertEqual(show.unhandled, frozenset({})) stream.seek(0) output = stream.read() @@ -438,44 +438,44 @@ def testShowPipeline(self): self.assertIn("class: lsst.pipe.base.tests.simpleQGraph.AddTask", output) # pipeline show = ShowInfo(["pipeline", "uri"], stream=stream) - show.show_pipeline_info(pipeline) + show.show_pipeline_info(pipeline, None) self.assertEqual(show.unhandled, frozenset({"uri"})) self.assertEqual(show.handled, {"pipeline"}) stream = StringIO() show = ShowInfo(["config=task::addend.missing"], stream=stream) # No match - show.show_pipeline_info(pipeline) + show.show_pipeline_info(pipeline, None) stream.seek(0) output = stream.read().strip() self.assertEqual("### Configuration for task `task'", output) stream = StringIO() show = ShowInfo(["config=task::addEnd:NOIGNORECASE"], stream=stream) # No match - show.show_pipeline_info(pipeline) + show.show_pipeline_info(pipeline, None) stream.seek(0) output = stream.read().strip() self.assertEqual("### Configuration for task `task'", output) stream = StringIO() show = ShowInfo(["config=task::addEnd"], stream=stream) # Match but warns - show.show_pipeline_info(pipeline) + show.show_pipeline_info(pipeline, None) stream.seek(0) output = stream.read().strip() self.assertIn("NOIGNORECASE", output) show = ShowInfo(["dump-config=notask"]) with self.assertRaises(ValueError) as cm: - show.show_pipeline_info(pipeline) + show.show_pipeline_info(pipeline, None) self.assertIn("Pipeline has no tasks named notask", str(cm.exception)) show = ShowInfo(["history"]) with self.assertRaises(ValueError) as cm: - show.show_pipeline_info(pipeline) + show.show_pipeline_info(pipeline, None) self.assertIn("Please provide a value", str(cm.exception)) show = ShowInfo(["history=notask::param"]) with self.assertRaises(ValueError) as cm: - show.show_pipeline_info(pipeline) + show.show_pipeline_info(pipeline, None) self.assertIn("Pipeline has no tasks named notask", str(cm.exception)) def test_execution_resources_parameters(self) -> None: From 878407c1ae514bf75076d272284a4b37db8c7eab Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Fri, 4 Aug 2023 15:29:10 -0400 Subject: [PATCH 4/6] Add changelog entry. --- doc/changes/DM-39779.feature.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/DM-39779.feature.md diff --git a/doc/changes/DM-39779.feature.md b/doc/changes/DM-39779.feature.md new file mode 100644 index 00000000..3eb205f4 --- /dev/null +++ b/doc/changes/DM-39779.feature.md @@ -0,0 +1 @@ +Add `pipeline-graph` and `task-graph` options for `pipetask build --show`, which provide text-art visualization of pipeline graphs. From 27d11c90da68fe64cf82d56819762be4407d360b Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Sat, 9 Sep 2023 09:10:30 -0400 Subject: [PATCH 5/6] Add tests for show pipeline-graph and show task-graph. --- tests/test_cmdLineFwk.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_cmdLineFwk.py b/tests/test_cmdLineFwk.py index 4c9a35f4..1b5f1db1 100644 --- a/tests/test_cmdLineFwk.py +++ b/tests/test_cmdLineFwk.py @@ -456,6 +456,31 @@ def testShowPipeline(self): output = stream.read().strip() self.assertEqual("### Configuration for task `task'", output) + stream = StringIO() + show = ShowInfo(["pipeline-graph"], stream=stream) # No match + show.show_pipeline_info(pipeline, None) + stream.seek(0) + output = stream.read().strip() + self.assertEqual( + "\n".join( + [ + "○ add_dataset_in", + "│", + "■ task", + "│", + "◍ add_dataset_out, add2_dataset_out", + ] + ), + output, + ) + + stream = StringIO() + show = ShowInfo(["task-graph"], stream=stream) # No match + show.show_pipeline_info(pipeline, None) + stream.seek(0) + output = stream.read().strip() + self.assertEqual("■ task", output) + stream = StringIO() show = ShowInfo(["config=task::addEnd"], stream=stream) # Match but warns show.show_pipeline_info(pipeline, None) From 01b3285a323277a7ae77a1a84622c676f01ffaeb Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Tue, 12 Sep 2023 16:11:33 -0400 Subject: [PATCH 6/6] Guard against confusion of passing -b when it won't be used. --- python/lsst/ctrl/mpexec/cli/cmd/commands.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/lsst/ctrl/mpexec/cli/cmd/commands.py b/python/lsst/ctrl/mpexec/cli/cmd/commands.py index 7dce8d01..4f1931b2 100644 --- a/python/lsst/ctrl/mpexec/cli/cmd/commands.py +++ b/python/lsst/ctrl/mpexec/cli/cmd/commands.py @@ -123,6 +123,11 @@ def build(ctx: click.Context, **kwargs: Any) -> None: """ kwargs = _collectActions(ctx, **kwargs) show = ShowInfo(kwargs.pop("show", [])) + if kwargs.get("butler_config") is not None and {"pipeline-graph", "task-graph"}.isdisjoint(show.commands): + raise click.ClickException( + "--butler-config was provided but nothing uses it " + "(only --show pipeline-graph and --show task-graph do)." + ) script.build(**kwargs, show=show) _unhandledShow(show, "build")