From f3c379f659c80fa583eb559ca16a2dcceb813b57 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Fri, 18 Aug 2023 11:31:34 -0400 Subject: [PATCH] Update tests to make skip-existing-in scenarios more realistic. The new QG generation algorithm in pipe_base now expects all init-output datasets to be present in the skip-existing-in collections when all quanta for that task are skipped because their regular outputs (metadata in particular) exist in those collections. --- tests/test_cmdLineFwk.py | 1 + tests/test_separablePipelineExecutor.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_cmdLineFwk.py b/tests/test_cmdLineFwk.py index ccb8906d..ce5064c2 100644 --- a/tests/test_cmdLineFwk.py +++ b/tests/test_cmdLineFwk.py @@ -725,6 +725,7 @@ def testSimpleQGraphSkipExisting_outputs(self): "add_init_output1", "task0_metadata", "task0_log", + "task0_config", ], }, ) diff --git a/tests/test_separablePipelineExecutor.py b/tests/test_separablePipelineExecutor.py index 19a4ea22..c42a91ae 100644 --- a/tests/test_separablePipelineExecutor.py +++ b/tests/test_separablePipelineExecutor.py @@ -26,6 +26,7 @@ import lsst.daf.butler import lsst.daf.butler.tests as butlerTests +import lsst.pex.config import lsst.utils.tests from lsst.ctrl.mpexec import SeparablePipelineExecutor from lsst.pipe.base import Instrument, Pipeline, PipelineDatasetTypes, TaskMetadata @@ -62,6 +63,7 @@ def setUp(self): butlerTests.addDatasetType(self.butler, "intermediate", set(), "StructuredDataDict") butlerTests.addDatasetType(self.butler, "a_log", set(), "ButlerLogRecords") butlerTests.addDatasetType(self.butler, "a_metadata", set(), "TaskMetadata") + butlerTests.addDatasetType(self.butler, "a_config", set(), "Config") def test_pre_execute_qgraph(self): # Too hard to make a quantum graph from scratch. @@ -71,7 +73,6 @@ def test_pre_execute_qgraph(self): graph = executor.make_quantum_graph(pipeline) butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict") - butlerTests.addDatasetType(self.butler, "a_config", set(), "Config") butlerTests.addDatasetType(self.butler, "b_config", set(), "Config") butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords") butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata") @@ -100,7 +101,6 @@ def test_pre_execute_qgraph_unconnected(self): graph = executor.make_quantum_graph(pipeline) butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict") - butlerTests.addDatasetType(self.butler, "a_config", set(), "Config") butlerTests.addDatasetType(self.butler, "b_config", set(), "Config") butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords") butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata") @@ -120,7 +120,6 @@ def test_pre_execute_qgraph_empty(self): graph = lsst.pipe.base.QuantumGraph({}, universe=self.butler.dimensions) butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict") - butlerTests.addDatasetType(self.butler, "a_config", set(), "Config") butlerTests.addDatasetType(self.butler, "b_config", set(), "Config") butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords") butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata") @@ -163,7 +162,6 @@ def test_pre_execute_qgraph_init_outputs(self): graph = executor.make_quantum_graph(pipeline) butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict") - butlerTests.addDatasetType(self.butler, "a_config", set(), "Config") butlerTests.addDatasetType(self.butler, "b_config", set(), "Config") butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords") butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata") @@ -185,7 +183,6 @@ def test_pre_execute_qgraph_versions(self): graph = executor.make_quantum_graph(pipeline) butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict") - butlerTests.addDatasetType(self.butler, "a_config", set(), "Config") butlerTests.addDatasetType(self.butler, "b_config", set(), "Config") butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords") butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata") @@ -290,6 +287,7 @@ def test_make_quantum_graph_nowhere_skiptotal_noclobber(self): self.butler.put({"zero": 0}, "intermediate") self.butler.put(lsst.daf.butler.ButlerLogRecords.from_records([]), "a_log") self.butler.put(TaskMetadata(), "a_metadata") + self.butler.put(lsst.pex.config.Config(), "a_config") graph = executor.make_quantum_graph(pipeline) self.assertTrue(graph.isConnected) @@ -366,6 +364,7 @@ def test_make_quantum_graph_nowhere_skiptotal_clobber(self): self.butler.put({"zero": 0}, "intermediate") self.butler.put(lsst.daf.butler.ButlerLogRecords.from_records([]), "a_log") self.butler.put(TaskMetadata(), "a_metadata") + self.butler.put(lsst.pex.config.Config(), "a_config") graph = executor.make_quantum_graph(pipeline) self.assertTrue(graph.isConnected) @@ -404,6 +403,7 @@ def test_make_quantum_graph_alloutput_skip(self): butlerTests.addDatasetType(self.butler, "output", set(), "StructuredDataDict") butlerTests.addDatasetType(self.butler, "b_log", set(), "ButlerLogRecords") butlerTests.addDatasetType(self.butler, "b_metadata", set(), "TaskMetadata") + butlerTests.addDatasetType(self.butler, "b_config", set(), "Config") self.butler.put({"zero": 0}, "input") self.butler.put({"zero": 0}, "intermediate") @@ -412,6 +412,8 @@ def test_make_quantum_graph_alloutput_skip(self): self.butler.put({"zero": 0}, "output") self.butler.put(lsst.daf.butler.ButlerLogRecords.from_records([]), "b_log") self.butler.put(TaskMetadata(), "b_metadata") + self.butler.put(lsst.pex.config.Config(), "a_config") + self.butler.put(lsst.pex.config.Config(), "b_config") graph = executor.make_quantum_graph(pipeline) self.assertEqual(len(graph), 0) @@ -482,6 +484,7 @@ def test_run_pipeline_skiptotal_noclobber(self): self.butler.put({"zero": 0}, "intermediate") self.butler.put(lsst.daf.butler.ButlerLogRecords.from_records([]), "a_log") self.butler.put(TaskMetadata(), "a_metadata") + self.butler.put(lsst.pex.config.Config(), "a_config") graph = executor.make_quantum_graph(pipeline) executor.pre_execute_qgraph( graph, @@ -564,6 +567,7 @@ def test_run_pipeline_skiptotal_clobber_connected(self): self.butler.put({"zero": 0}, "intermediate") self.butler.put(lsst.daf.butler.ButlerLogRecords.from_records([]), "a_log") self.butler.put(TaskMetadata(), "a_metadata") + self.butler.put(lsst.pex.config.Config(), "a_config") graph = executor.make_quantum_graph(pipeline) executor.pre_execute_qgraph( graph,