Update tests to make skip-existing-in scenarios more realistic.

The new QG generation algorithm in pipe_base now expects all init-output datasets to be present in the skip-existing-in collections when all quanta for that task are skipped because their regular outputs (metadata in particular) exist in those collections.
lsst · Aug 21, 2023 · f3c379f · f3c379f
1 parent 3ef28a6
commit f3c379f
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 5 deletions.
diff --git a/tests/test_cmdLineFwk.py b/tests/test_cmdLineFwk.py
@@ -725,6 +725,7 @@ def testSimpleQGraphSkipExisting_outputs(self):
                     "add_init_output1",
                     "task0_metadata",
                     "task0_log",
+                    "task0_config",
                 ],
             },
         )

diff --git a/tests/test_separablePipelineExecutor.py b/tests/test_separablePipelineExecutor.py
@@ -26,6 +26,7 @@
 
 import lsst.daf.butler
 import lsst.daf.butler.tests as butlerTests
+import lsst.pex.config
 import lsst.utils.tests
 from lsst.ctrl.mpexec import SeparablePipelineExecutor
 from lsst.pipe.base import Instrument, Pipeline, PipelineDatasetTypes, TaskMetadata
@@ -62,6 +63,7 @@ def setUp(self):
         butlerTests.addDatasetType(self.butler, "intermediate", set(), "StructuredDataDict")
         butlerTests.addDatasetType(self.butler, "a_log", set(), "ButlerLogRecords")
         butlerTests.addDatasetType(self.butler, "a_metadata", set(), "TaskMetadata")
+        butlerTests.addDatasetType(self.butler, "a_config", set(), "Config")
 
     def test_pre_execute_qgraph(self):
         # Too hard to make a quantum graph from scratch.
@@ -71,7 +73,6 @@ def test_pre_execute_qgraph(self):
         graph = executor.make_quantum_graph(pipeline)
 
         butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict")
-        butlerTests.addDatasetType(self.butler, "a_config", set(), "Config")
         butlerTests.addDatasetType(self.butler, "b_config", set(), "Config")
         butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords")
         butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata")
@@ -100,7 +101,6 @@ def test_pre_execute_qgraph_unconnected(self):
         graph = executor.make_quantum_graph(pipeline)
 
         butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict")
-        butlerTests.addDatasetType(self.butler, "a_config", set(), "Config")
         butlerTests.addDatasetType(self.butler, "b_config", set(), "Config")
         butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords")
         butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata")
@@ -120,7 +120,6 @@ def test_pre_execute_qgraph_empty(self):
         graph = lsst.pipe.base.QuantumGraph({}, universe=self.butler.dimensions)
 
         butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict")
-        butlerTests.addDatasetType(self.butler, "a_config", set(), "Config")
         butlerTests.addDatasetType(self.butler, "b_config", set(), "Config")
         butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords")
         butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata")
@@ -163,7 +162,6 @@ def test_pre_execute_qgraph_init_outputs(self):
         graph = executor.make_quantum_graph(pipeline)
 
         butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict")
-        butlerTests.addDatasetType(self.butler, "a_config", set(), "Config")
         butlerTests.addDatasetType(self.butler, "b_config", set(), "Config")
         butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords")
         butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata")
@@ -185,7 +183,6 @@ def test_pre_execute_qgraph_versions(self):
         graph = executor.make_quantum_graph(pipeline)
 
         butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict")
-        butlerTests.addDatasetType(self.butler, "a_config", set(), "Config")
         butlerTests.addDatasetType(self.butler, "b_config", set(), "Config")
         butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords")
         butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata")
@@ -290,6 +287,7 @@ def test_make_quantum_graph_nowhere_skiptotal_noclobber(self):
         self.butler.put({"zero": 0}, "intermediate")
         self.butler.put(lsst.daf.butler.ButlerLogRecords.from_records([]), "a_log")
         self.butler.put(TaskMetadata(), "a_metadata")
+        self.butler.put(lsst.pex.config.Config(), "a_config")
 
         graph = executor.make_quantum_graph(pipeline)
         self.assertTrue(graph.isConnected)
@@ -366,6 +364,7 @@ def test_make_quantum_graph_nowhere_skiptotal_clobber(self):
         self.butler.put({"zero": 0}, "intermediate")
         self.butler.put(lsst.daf.butler.ButlerLogRecords.from_records([]), "a_log")
         self.butler.put(TaskMetadata(), "a_metadata")
+        self.butler.put(lsst.pex.config.Config(), "a_config")
 
         graph = executor.make_quantum_graph(pipeline)
         self.assertTrue(graph.isConnected)
@@ -404,6 +403,7 @@ def test_make_quantum_graph_alloutput_skip(self):
         butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict")
         butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords")
         butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata")
+        butlerTests.addDatasetType(self.butler, "b_config", set(), "Config")
 
         self.butler.put({"zero": 0}, "input")
         self.butler.put({"zero": 0}, "intermediate")
@@ -412,6 +412,8 @@ def test_make_quantum_graph_alloutput_skip(self):
         self.butler.put({"zero": 0}, "output")
         self.butler.put(lsst.daf.butler.ButlerLogRecords.from_records([]), "b_log")
         self.butler.put(TaskMetadata(), "b_metadata")
+        self.butler.put(lsst.pex.config.Config(), "a_config")
+        self.butler.put(lsst.pex.config.Config(), "b_config")
 
         graph = executor.make_quantum_graph(pipeline)
         self.assertEqual(len(graph), 0)
@@ -482,6 +484,7 @@ def test_run_pipeline_skiptotal_noclobber(self):
         self.butler.put({"zero": 0}, "intermediate")
         self.butler.put(lsst.daf.butler.ButlerLogRecords.from_records([]), "a_log")
         self.butler.put(TaskMetadata(), "a_metadata")
+        self.butler.put(lsst.pex.config.Config(), "a_config")
         graph = executor.make_quantum_graph(pipeline)
         executor.pre_execute_qgraph(
             graph,
@@ -564,6 +567,7 @@ def test_run_pipeline_skiptotal_clobber_connected(self):
         self.butler.put({"zero": 0}, "intermediate")
         self.butler.put(lsst.daf.butler.ButlerLogRecords.from_records([]), "a_log")
         self.butler.put(TaskMetadata(), "a_metadata")
+        self.butler.put(lsst.pex.config.Config(), "a_config")
         graph = executor.make_quantum_graph(pipeline)
         executor.pre_execute_qgraph(
             graph,