Merge pull request #309 from lsst/tickets/DM-38065-v24

DM-38065: V24.1 release notes

.pre-commit-config.yaml

@@ -1,12 +1,13 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v4.4.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
       - id: trailing-whitespace
+      - id: check-toml
   - repo: https://github.com/psf/black
-    rev: 22.1.0
+    rev: 23.1.0
     hooks:
       - id: black
         # It is recommended to specify the latest version of Python
@@ -15,7 +16,11 @@ repos:
         # https://pre-commit.com/#top_level-default_language_version
         language_version: python3.10
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
         name: isort (python)
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8

doc/lsst.pipe.base/CHANGES.rst

@@ -1,3 +1,23 @@
+lsst-pipe-base v24.1.0 (2023-02-23)
+===================================
+
+New Features
+------------
+
+- ``QuantumGraph`` now saves the ``DimensionUniverse`` it was created with when it is persisted. This removes the need
+  to explicitly pass the ``DimensionUniverse`` when loading a saved graph. (`DM-35082 <https://jira.lsstcorp.org/browse/DM-35082>`_)
+- Allow ``PipelineTasks`` to provide defaults for the ``--dataset-query-constraints`` option for the ``pipetask`` tool. (`DM-37786 <https://jira.lsstcorp.org/browse/DM-37786>`_)
+
+
+Bug Fixes
+---------
+
+- Ensure ``QuantumGraphs`` are given a ``DimensionUniverse`` at construction.
+
+  This fixes a mostly-spurious dimension universe inconsistency warning when reading ``QuantumGraphs``, introduced on `DM-35082 <https://jira.lsstcorp.org/browse/DM-35082>`_. (`DM-35681 <https://jira.lsstcorp.org/browse/DM-35681>`_)
+- Fix an error message that says that repository state has changed during ``QuantumGraph`` generation when init input datasets are just missing. (`DM-37786 <https://jira.lsstcorp.org/browse/DM-37786>`_)
+
+
 lsst-pipe-base v24.0.0 (2022-08-26)
 ===================================
 

python/lsst/pipe/base/_task_metadata.py

@@ -169,7 +169,6 @@ def add(self, name: str, value: Any) -> None:
         keys = self._getKeys(name)
         key0 = keys.pop(0)
         if len(keys) == 0:
-
             # If add() is being used, always store the value in the arrays
             # property as a list. It's likely there will be another call.
             slot_type, value = self._validate_value(value)

python/lsst/pipe/base/connections.py

@@ -411,7 +411,7 @@ class attribute must match a function argument name in the ``run``
 
     dimensions: ClassVar[Set[str]]
 
-    def __init__(self, *, config: "PipelineTaskConfig" = None):
+    def __init__(self, *, config: "PipelineTaskConfig" | None = None):
         self.inputs: Set[str] = set(self.inputs)
         self.prerequisiteInputs: Set[str] = set(self.prerequisiteInputs)
         self.outputs: Set[str] = set(self.outputs)

python/lsst/pipe/base/graph/_implDetails.py

@@ -130,8 +130,8 @@ def removeConsumer(self, key: _T, value: _U) -> None:
         if (result := self._consumers.get(key)) is not None:
             result.discard(value)
         if self._createInverse:
-            if result := self._itemsDict.get(value):
-                result.discard(key)
+            if result_inverse := self._itemsDict.get(value):
+                result_inverse.discard(key)
 
     def getConsumers(self, key: _T) -> Set[_U]:
         """Return all values associated with the consumption of the supplied

python/lsst/pipe/base/graph/_loadHelpers.py

@@ -404,7 +404,7 @@ def _determineLoader(self) -> Type[DefaultLoadHelper]:
         # that IOBase and BinaryIO actually have incompatible method
         # signatures.  IOBase *is* a base class of what open(..., 'rb')
         # returns, so it's what we have to use at runtime.
-        if isinstance(self.uri, io.IOBase):  # type: ignore
+        if isinstance(self.uri, io.IOBase):
             key = BinaryIO
         else:
             key = type(self.uri)

python/lsst/pipe/base/pipeTools.py

@@ -102,7 +102,6 @@ def isPipelineOrdered(
     # Build a map of DatasetType name to producer's index in a pipeline
     producerIndex = {}
     for idx, taskDef in enumerate(pipeline):
-
         for attr in iterConnections(taskDef.connections, "outputs"):
             if attr.name in producerIndex:
                 raise DuplicateOutputError(
@@ -112,7 +111,6 @@ def isPipelineOrdered(
 
     # check all inputs that are also someone's outputs
     for idx, taskDef in enumerate(pipeline):
-
         # get task input DatasetTypes, this can only be done via class method
         inputs = {name: getattr(taskDef.connections, name) for name in taskDef.connections.inputs}
         for dsTypeDescr in inputs.values():
@@ -180,7 +178,6 @@ def orderPipeline(pipeline: List[TaskDef]) -> List[TaskDef]:
     queue = [-1]
     result = []
     while queue:
-
         # move to final list, drop -1
         idx = queue.pop(0)
         if idx >= 0:

python/lsst/pipe/base/pipeline.py

@@ -1289,7 +1289,6 @@ def initOutputNames(
             pipeline = pipeline.toExpandedPipeline()
 
         for taskDef in pipeline:
-
             # all task InitOutputs
             for name in taskDef.connections.initOutputs:
                 attribute = getattr(taskDef.connections, name)

python/lsst/pipe/base/pipelineIR.py

@@ -30,7 +30,19 @@
 from collections import Counter
 from collections.abc import Iterable as abcIterable
 from dataclasses import dataclass, field
-from typing import Any, Dict, Generator, List, Literal, Mapping, MutableMapping, Optional, Set, Union
+from typing import (
+    Any,
+    Dict,
+    Generator,
+    Hashable,
+    List,
+    Literal,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Set,
+    Union,
+)
 
 import yaml
 from deprecated.sphinx import deprecated
@@ -47,7 +59,7 @@ class PipelineYamlLoader(yaml.SafeLoader):
     found inside a pipeline file at a given scope.
     """
 
-    def construct_mapping(self, node: yaml.Node, deep: bool = False) -> Mapping[str, Any]:
+    def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict[Hashable, Any]:
         # do the call to super first so that it can do all the other forms of
         # checking on this node. If you check the uniqueness of keys first
         # it would save the work that super does in the case of a failure, but

python/lsst/pipe/base/tests/simpleQGraph.py

@@ -128,7 +128,6 @@ class AddTask(PipelineTask):
     """Factory that makes instances"""
 
     def run(self, input: int) -> Struct:  # type: ignore
-
         if self.taskFactory:
             # do some bookkeeping
             if self.taskFactory.stopAt == self.taskFactory.countExec:
@@ -273,7 +272,7 @@ def makeSimpleButler(root: str, run: str = "test", inMemory: bool = True) -> But
 
 
 def populateButler(
-    pipeline: Pipeline, butler: Butler, datasetTypes: Dict[Optional[str], List[str]] = None
+    pipeline: Pipeline, butler: Butler, datasetTypes: Dict[Optional[str], List[str]] | None = None
 ) -> None:
     """Populate data butler with data needed for test.
 

requirements.txt

@@ -2,10 +2,11 @@ pyyaml >= 5.1
 pydantic
 numpy >= 1.17
 networkx
-git+https://github.com/lsst/daf_butler@main#egg=lsst-daf-butler
-git+https://github.com/lsst/utils@main#egg=lsst-utils
-git+https://github.com/lsst/resources@main#egg=lsst-resources
-git+https://github.com/lsst/pex_config@main#egg=lsst-pex-config
+git+https://github.com/lsst/[email protected]#egg=lsst-daf-butler
+git+https://github.com/lsst/[email protected]#egg=lsst-utils
+git+https://github.com/lsst/[email protected]#egg=lsst-resources
+git+https://github.com/lsst/[email protected]#egg=lsst-pex-config
+git+https://github.com/lsst/[email protected]#egg=lsst-sphgeom
 # For now a S3 class is loaded unconditionally
 boto3
 botocore

setup.cfg

@@ -24,10 +24,10 @@ packages=find:
 setup_requires =
   setuptools >=46.0
 install_requires =
-  lsst-resources
-  lsst-utils
-  lsst-daf-butler
-  lsst-pex-config
+  lsst-resources >=24.0.0,<24.100
+  lsst-utils >=24.0.0,<24.100
+  lsst-daf-butler >=24.0.0,<24.100
+  lsst-pex-config >=24.0.0,<24.100
   pydantic
   networkx
   pyyaml >= 5.1

tests/test_cliCmdRegisterInstrument.py

@@ -29,7 +29,6 @@
 
 
 class RegisterInstrumentTest(CliCmdTestBase, unittest.TestCase):
-
     mockFuncName = "lsst.pipe.base.cli.cmd.commands.script.register_instrument"
 
     @staticmethod