From d80de8e12b22f38bb9e429f9afd11f41f93292cf Mon Sep 17 00:00:00 2001
From: Jim Bosch <jbosch@astro.princeton.edu>
Date: Fri, 3 Dec 2021 11:26:59 -0500
Subject: [PATCH] Add pipetask option to save pipelines in expanded form.

---
 python/lsst/ctrl/mpexec/cli/opt/optionGroups.py |  1 +
 python/lsst/ctrl/mpexec/cli/opt/options.py      | 10 ++++++++++
 python/lsst/ctrl/mpexec/cli/script/build.py     |  6 ++++++
 python/lsst/ctrl/mpexec/cmdLineFwk.py           |  3 +++
 tests/test_cliScript.py                         | 15 +++++++++++++++
 5 files changed, 35 insertions(+)

diff --git a/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py b/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py
index d68cc3f9..1b931a76 100644
--- a/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py
+++ b/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py
@@ -73,6 +73,7 @@ def __init__(self) -> None:
             ),
             ctrlMpExecOpts.order_pipeline_option(),
             ctrlMpExecOpts.save_pipeline_option(),
+            ctrlMpExecOpts.expand_pipeline_option(),
             ctrlMpExecOpts.pipeline_dot_option(),
             pipeBaseOpts.instrument_option(help=instrumentOptionHelp, metavar="instrument", multiple=True),
             ctrlMpExecOpts.butler_config_option(required=False),
diff --git a/python/lsst/ctrl/mpexec/cli/opt/options.py b/python/lsst/ctrl/mpexec/cli/opt/options.py
index abc098cf..570b07c9 100644
--- a/python/lsst/ctrl/mpexec/cli/opt/options.py
+++ b/python/lsst/ctrl/mpexec/cli/opt/options.py
@@ -287,6 +287,16 @@
     type=MWPath(dir_okay=False, file_okay=True, writable=True),
 )
 
+expand_pipeline_option = MWOptionDecorator(
+    "-s",
+    "--expand-pipeline",
+    help=unwrap(
+        """Directory location for storing the fully-expanded
+                                                       pipeline definition."""
+    ),
+    type=MWPath(dir_okay=True, file_okay=False, writable=True),
+)
+
 save_qgraph_option = MWOptionDecorator(
     "-q",
     "--save-qgraph",
diff --git a/python/lsst/ctrl/mpexec/cli/script/build.py b/python/lsst/ctrl/mpexec/cli/script/build.py
index 7b00d333..e96d89d5 100644
--- a/python/lsst/ctrl/mpexec/cli/script/build.py
+++ b/python/lsst/ctrl/mpexec/cli/script/build.py
@@ -40,6 +40,7 @@ def build(  # type: ignore
     pipeline_actions,
     pipeline_dot,
     save_pipeline,
+    expand_pipeline,
     show,
     butler_config=None,
     **kwargs,
@@ -67,6 +68,10 @@ def build(  # type: ignore
         Path location for storing GraphViz DOT representation of a pipeline.
     save_pipeline : `str`
         Path location for storing resulting pipeline definition in YAML format.
+    expand_pipeline : `str`
+        Directory path location for storing the expanded pipeline definition,
+        with all references to other files resolved and written to config files
+        in the directory.
     show : `lsst.ctrl.mpexec.showInfo.ShowInfo`
         Descriptions of what to dump to stdout.
     butler_config : `str`, `dict`, or `lsst.daf.butler.Config`, optional
@@ -103,6 +108,7 @@ def build(  # type: ignore
         pipeline_actions=pipeline_actions,
         pipeline_dot=pipeline_dot,
         save_pipeline=save_pipeline,
+        expand_pipeline=expand_pipeline,
     )
 
     f = CmdLineFwk()
diff --git a/python/lsst/ctrl/mpexec/cmdLineFwk.py b/python/lsst/ctrl/mpexec/cmdLineFwk.py
index c085e8a3..7d9f0bcf 100644
--- a/python/lsst/ctrl/mpexec/cmdLineFwk.py
+++ b/python/lsst/ctrl/mpexec/cmdLineFwk.py
@@ -582,6 +582,9 @@ def makePipeline(self, args: SimpleNamespace) -> Pipeline:
         if args.save_pipeline:
             pipeline.write_to_uri(args.save_pipeline)
 
+        if args.expand_pipeline:
+            pipeline.write_to_uri(args.expand_pipeline, expand=True)
+
         return pipeline
 
     def makeGraph(self, pipeline: Pipeline, args: SimpleNamespace) -> QuantumGraph | None:
diff --git a/tests/test_cliScript.py b/tests/test_cliScript.py
index f05c4771..e31a2ae6 100644
--- a/tests/test_cliScript.py
+++ b/tests/test_cliScript.py
@@ -52,6 +52,7 @@ def buildArgs(**kwargs):
             pipeline_actions=(),
             pipeline_dot=None,
             save_pipeline=None,
+            expand_pipeline=None,
             show=ShowInfo([]),
         )
         defaultArgs.update(kwargs)
@@ -77,6 +78,20 @@ def testSavePipeline(self):
             self.assertIsInstance(pipeline, Pipeline)
             self.assertEqual(len(pipeline), 0)
 
+    def testExpandPipeline(self):
+        """Test expanded pipeline serialization."""
+        with tempfile.TemporaryDirectory() as tempdir:
+            # make empty pipeline and store it in a directory
+            directory = os.path.join(tempdir, "pipeline_dir")
+            pipeline = script.build(**self.buildArgs(expand_pipeline=directory))
+            self.assertIsInstance(pipeline, Pipeline)
+            self.assertTrue(os.path.isdir(directory))
+            # read pipeline from a directory
+            pipeline = script.build(**self.buildArgs(pipeline=directory))
+            self.assertIsInstance(pipeline, Pipeline)
+            self.assertIsInstance(pipeline, Pipeline)
+            self.assertEqual(len(pipeline), 0)
+
     def testShowPipeline(self):
         """Test showing the pipeline."""