Merge pull request #295 from lsst/tickets/DM-45113

DM-45113: Re-enable skipping and clobbering with LimitedButler

python/lsst/ctrl/mpexec/cmdLineFwk.py

@@ -994,8 +994,7 @@ def runGraphQBB(self, task_factory: TaskFactory, args: SimpleNamespace) -> None:
             exitOnKnownError=args.fail_fast,
             limited_butler_factory=_butler_factory,
             resources=resources,
-            clobberOutputs=True,
-            skipExisting=True,
+            assumeNoExistingOutputs=True,
         )
 
         timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout

python/lsst/ctrl/mpexec/dotTools.py

@@ -113,7 +113,7 @@ def _renderQuantumNode(
     labels = [f"{quantumNode.nodeId}", html.escape(taskDef.label)]
     dataId = quantumNode.quantum.dataId
     assert dataId is not None, "Quantum DataId cannot be None"
-    labels.extend(f"{key} = {dataId[key]}" for key in sorted(dataId.keys()))
+    labels.extend(f"{key} = {dataId[key]}" for key in sorted(dataId.required.keys()))
     _renderNode(file, nodeName, "quantum", labels)
 
 
@@ -128,7 +128,7 @@ def _renderDSTypeNode(name: str, dimensions: list[str], file: io.TextIOBase) ->
 def _renderDSNode(nodeName: str, dsRef: DatasetRef, file: io.TextIOBase) -> None:
     """Render GV node for a dataset"""
     labels = [html.escape(dsRef.datasetType.name), f"run: {dsRef.run!r}"]
-    labels.extend(f"{key} = {dsRef.dataId[key]}" for key in sorted(dsRef.dataId.keys()))
+    labels.extend(f"{key} = {dsRef.dataId[key]}" for key in sorted(dsRef.dataId.required.keys()))
     _renderNode(file, nodeName, "dataset", labels)
 
 
@@ -144,7 +144,7 @@ def _renderEdge(fromName: str, toName: str, file: io.TextIOBase, **kwargs: Any)
 def _datasetRefId(dsRef: DatasetRef) -> str:
     """Make an identifying string for given ref"""
     dsId = [dsRef.datasetType.name]
-    dsId.extend(f"{key} = {dsRef.dataId[key]}" for key in sorted(dsRef.dataId.keys()))
+    dsId.extend(f"{key} = {dsRef.dataId[key]}" for key in sorted(dsRef.dataId.required.keys()))
     return ":".join(dsId)
 
 

python/lsst/ctrl/mpexec/singleQuantumExecutor.py

@@ -123,6 +123,12 @@ class SingleQuantumExecutor(QuantumExecutor):
         Unlike ``skipExistingIn``, this works with limited butlers as well as
         full butlers.  Always set to `True` if ``skipExistingIn`` matches
         ``butler.run``.
+    assumeNoExistingOutputs : `bool`, optional
+        If `True`, assume preexisting outputs are impossible (e.g. because this
+        is known by higher-level code to be a new ``RUN`` collection), and do
+        not look for them.  This causes the ``skipExisting`` and
+        ``clobberOutputs`` options to be ignored, but unlike just setting both
+        of those to `False`, it also avoids all dataset existence checks.
     """
 
     def __init__(
@@ -136,6 +142,7 @@ def __init__(
         limited_butler_factory: Callable[[Quantum], LimitedButler] | None = None,
         resources: ExecutionResources | None = None,
         skipExisting: bool = False,
+        assumeNoExistingOutputs: bool = False,
     ):
         self.butler = butler
         self.taskFactory = taskFactory
@@ -144,6 +151,7 @@ def __init__(
         self.exitOnKnownError = exitOnKnownError
         self.limited_butler_factory = limited_butler_factory
         self.resources = resources
+        self.assumeNoExistingOutputs = assumeNoExistingOutputs
 
         if self.butler is None:
             assert limited_butler_factory is not None, "limited_butler_factory is needed when butler is None"
@@ -345,8 +353,7 @@ def checkExistingOutputs(
         """
         task_node = self._conform_task_def(task_node)
 
-        if not self.butler:
-            # Skip/prune logic only works for full butler.
+        if self.assumeNoExistingOutputs:
             return False
 
         if self.skipExisting: