From 53d339a2cb9feb20032120b02349e74ae0b8d461 Mon Sep 17 00:00:00 2001 From: Orion Eiger Date: Fri, 20 Sep 2024 21:03:24 -0700 Subject: [PATCH] Add check for graphs passed out of order and clarify docs --- python/lsst/ctrl/mpexec/cli/cmd/commands.py | 9 +++++--- python/lsst/ctrl/mpexec/cli/script/report.py | 24 +++++++++++++++++--- requirements.txt | 2 +- tests/test_cliCmdReport.py | 3 --- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/python/lsst/ctrl/mpexec/cli/cmd/commands.py b/python/lsst/ctrl/mpexec/cli/cmd/commands.py index edb2e3a2..5d26938a 100644 --- a/python/lsst/ctrl/mpexec/cli/cmd/commands.py +++ b/python/lsst/ctrl/mpexec/cli/cmd/commands.py @@ -373,7 +373,8 @@ def update_graph_run( is_flag=True, default=False, help="Use the QuantumProvenanceGraph instead of the QuantumGraphExecutionReport, " - "even when there is only one qgraph.", + "even when there is only one qgraph. Otherwise, the `QuantumGraphExecutionReport` " + "will run on one graph by default.", ) def report( repo: str, @@ -399,12 +400,14 @@ def report( Butler `collections` and `where` options are for use in `lsst.daf.butler.queryDatasets` if paring down the collections would be - useful. By default the collections and query be taken from the graphs. + useful. Pass collections in order of most to least recent. By default the + collections and query will be taken from the graphs. REPO is the location of the butler/registry config file. QGRAPHS is a `Sequence` of links to serialized Quantum Graphs which have - been executed and are to be analyzed. + been executed and are to be analyzed. Pass the graphs in order of first to + last executed. """ if any([force_v2, len(qgraphs) > 1, collections, where, curse_failed_logs]): script.report_v2( diff --git a/python/lsst/ctrl/mpexec/cli/script/report.py b/python/lsst/ctrl/mpexec/cli/script/report.py index db890e0e..62d88e79 100644 --- a/python/lsst/ctrl/mpexec/cli/script/report.py +++ b/python/lsst/ctrl/mpexec/cli/script/report.py @@ -136,9 +136,9 @@ def report_v2( butler_config : `str` The Butler used for this report. This should match the Butler used for the run associated with the executed quantum graph. - qgraph_uris : `Sequence[str]` + qgraph_uris : `Sequence` [`str`] One or more uris to the serialized Quantum Graph(s). - collections : `Sequence[str] | None` + collections : `Sequence` [`str`] | None` Collection(s) associated with said graphs/processing. For use in `lsst.daf.butler.registry.queryDatasets` if paring down the query would be useful. @@ -170,7 +170,25 @@ def report_v2( """ butler = Butler.from_config(butler_config, writeable=False) qpg = QuantumProvenanceGraph() - qgraphs = [QuantumGraph.loadUri(qgraph_uri) for qgraph_uri in qgraph_uris] + qgraphs = [] + for qgraph_uri in qgraph_uris: + qgraph = QuantumGraph.loadUri(qgraph_uri) + assert qgraph.metadata is not None, "Saved QGs always have metadata." + qgraphs.append(qgraph) + # If the most recent graph's timestamp was earlier than any of the + # previous graphs, raise a RuntimeError. + for count, qgraph in enumerate(qgraphs): + if len(qgraphs) > 1: + previous_graph = qgraphs[count - 1] + if qgraph.metadata["time"] < previous_graph.metadata["time"]: + raise RuntimeError( + f"""add_new_graph may only be called on graphs + which are passed in the order they were + created. Please call again, passing your + graphs in order. Time of first graph: + {qgraph.metadata["time"]} > + time of second graph: {previous_graph.metadata["time"]}""" + ) qpg.assemble_quantum_provenance_graph(butler, qgraphs, collections, where, curse_failed_logs) summary = qpg.to_summary(butler, do_store_logs=logs) print_summary(summary, full_output_filename, brief) diff --git a/requirements.txt b/requirements.txt index 025994df..4623a213 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,6 @@ networkx lsst-resources @ git+https://github.com/lsst/resources@main lsst-daf-butler @ git+https://github.com/lsst/daf_butler@main lsst-utils @ git+https://github.com/lsst/utils@main -lsst-pipe-base @ git+https://github.com/lsst/pipe_base@tickets/DM-41711 +lsst-pipe-base @ git+https://github.com/lsst/pipe_base@main lsst-pex-config @ git+https://github.com/lsst/pex_config@main sqlalchemy diff --git a/tests/test_cliCmdReport.py b/tests/test_cliCmdReport.py index 38b14c01..d3b8bb6b 100644 --- a/tests/test_cliCmdReport.py +++ b/tests/test_cliCmdReport.py @@ -83,9 +83,6 @@ def test_report(self): with open(test_filename) as f: report_output_dict = yaml.load(f, Loader=SafeLoader) - with open("delete_me.yaml", "w") as f: - yaml.safe_dump(report_output_dict, f) - self.assertIsNotNone(report_output_dict["task0"]) self.assertIsNotNone(report_output_dict["task0"]["failed_quanta"]) self.assertIsInstance(report_output_dict["task0"]["n_expected"], int)