From 7b141c3a0e82a40a8c4a9940afce0fa32e45bf15 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Thu, 6 Jul 2023 15:54:35 -0700 Subject: [PATCH 1/7] Change use of butler.datastore now that it's deprecated --- python/lsst/pipe/base/executionButlerBuilder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lsst/pipe/base/executionButlerBuilder.py b/python/lsst/pipe/base/executionButlerBuilder.py index 84204cae8..d2a865948 100644 --- a/python/lsst/pipe/base/executionButlerBuilder.py +++ b/python/lsst/pipe/base/executionButlerBuilder.py @@ -168,7 +168,7 @@ def _accumulate( if ref.isComponent(): ref = ref.makeCompositeRef() check_refs.add(ref) - exist_map = butler.datastore.knows_these(check_refs) + exist_map = butler._exists_many(check_refs, full_check=False) existing_ids = set(ref.id for ref, exists in exist_map.items() if exists) del exist_map @@ -240,7 +240,7 @@ def _export(butler: Butler, collections: Iterable[str] | None, inserts: DataSetT # export/import BackendClass = get_class_of(butler._config["repo_transfer_formats", "yaml", "export"]) backend = BackendClass(yamlBuffer, universe=butler.dimensions) - exporter = RepoExportContext(butler.registry, butler.datastore, backend, directory=None, transfer=None) + exporter = RepoExportContext(butler.registry, butler._datastore, backend, directory=None, transfer=None) # Need to ensure that the dimension records for outputs are # transferred. From 908bdce40deb182499cac3e4c18f4b35962d4c28 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Thu, 6 Jul 2023 16:43:12 -0700 Subject: [PATCH 2/7] Remove use of deprecated parameter --- python/lsst/pipe/base/graph/_versionDeserializers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/lsst/pipe/base/graph/_versionDeserializers.py b/python/lsst/pipe/base/graph/_versionDeserializers.py index a93898d5b..39cf583cb 100644 --- a/python/lsst/pipe/base/graph/_versionDeserializers.py +++ b/python/lsst/pipe/base/graph/_versionDeserializers.py @@ -39,7 +39,6 @@ DatasetRef, DatasetType, DimensionConfig, - DimensionRecord, DimensionUniverse, Quantum, SerializedDimensionRecord, @@ -534,7 +533,6 @@ def constructGraph( container = {} datasetDict = _DatasetTracker[DatasetTypeName, TaskDef](createInverse=True) taskToQuantumNode: defaultdict[TaskDef, set[QuantumNode]] = defaultdict(set) - recontitutedDimensions: dict[int, tuple[str, DimensionRecord]] = {} initInputRefs: dict[TaskDef, list[DatasetRef]] = {} initOutputRefs: dict[TaskDef, list[DatasetRef]] = {} @@ -610,7 +608,7 @@ def constructGraph( # reconstitute the node, passing in the dictionaries for the # loaded TaskDefs and dimension records. These are used to ensure # that each unique record is only loaded once - qnode = QuantumNode.from_simple(nodeDeserialized, loadedTaskDef, universe, recontitutedDimensions) + qnode = QuantumNode.from_simple(nodeDeserialized, loadedTaskDef, universe) container[qnode.nodeId] = qnode taskToQuantumNode[loadedTaskDef[nodeTaskLabel]].add(qnode) From d9308725d691d0831ade72bf2858c5ecda6c89ae Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Fri, 7 Jul 2023 16:58:11 -0700 Subject: [PATCH 3/7] Run black on the python documentation examples --- .../creating-a-pipelinetask.rst | 343 ++++++++++-------- doc/lsst.pipe.base/creating-a-task.rst | 14 +- doc/lsst.pipe.base/task-retargeting-howto.rst | 2 + .../testing-a-pipeline-task.rst | 79 ++-- 4 files changed, 246 insertions(+), 192 deletions(-) diff --git a/doc/lsst.pipe.base/creating-a-pipelinetask.rst b/doc/lsst.pipe.base/creating-a-pipelinetask.rst index 2a98f043e..90e36f4d9 100644 --- a/doc/lsst.pipe.base/creating-a-pipelinetask.rst +++ b/doc/lsst.pipe.base/creating-a-pipelinetask.rst @@ -35,7 +35,6 @@ Next, create a |Task| class that performs the measurements: .. code-block:: python class ApertureTask(pipeBase.Task): - ConfigClass = ApertureTaskConfig _DefaultName = "apertureDemoTask" @@ -44,15 +43,17 @@ Next, create a |Task| class that performs the measurements: self.apRad = self.config.apRad self.outputSchema = afwTable.SourceTable.makeMinimalSchema() - self.apKey = self.outputSchema.addField("apFlux", type=np.float64, - doc="Ap flux measured") + self.apKey = self.outputSchema.addField( + "apFlux", type=np.float64, doc="Ap flux measured" + ) self.outputCatalog = afwTable.SourceCatalog(self.outputSchema) - def run(self, exposure: afwImage.Exposure, - inputCatalog: afwTable.SourceCatalog) -> pipeBase.Struct: + def run( + self, exposure: afwImage.Exposure, inputCatalog: afwTable.SourceCatalog + ) -> pipeBase.Struct: # set dimension cutouts to 3 times the apRad times 2 (for diameter) - dimensions = (3*self.apRad*2, 3*self.apRad*2) + dimensions = (3 * self.apRad * 2, 3 * self.apRad * 2) # Get indexes for each pixel indy, indx = np.indices(dimensions) @@ -63,11 +64,14 @@ Next, create a |Task| class that performs the measurements: center = Point2I(source.getCentroid()) center = (center.getY(), center.getX()) - stamp = exposure.image.array[center[0]-3*self.apRad: center[0]+3*self.apRad, - center[1]-3*self.apRad: center[1]+3*self.apRad] - mask = ((indy - center[0])**2 - + (indx - center[0])**2)**0.5 < self.apRad - flux = np.sum(stamp*mask) + stamp = exposure.image.array[ + center[0] - 3 * self.apRad : center[0] + 3 * self.apRad, + center[1] - 3 * self.apRad : center[1] + 3 * self.apRad, + ] + mask = ( + (indy - center[0]) ** 2 + (indx - center[0]) ** 2 + ) ** 0.5 < self.apRad + flux = np.sum(stamp * mask) # Add a record to the output catalog tmpRecord = self.outputCatalog.addNew() @@ -103,24 +107,31 @@ task). .. code-block:: python - class ApertureTaskConnections(pipeBase.PipelineTaskConnections, - dimensions=("visit", "detector", "band")):, - exposure = connectionTypes.Input(doc="Input exposure to make measurements " - "on", - dimensions=("visit", "detector"), - storageClass="ExposureF", - name="calexp") - inputCatalog = connectionTypes.Input(doc="Input catalog with existing " - "measurements", - dimensions=("visit", "detector", - "band",), - storageClass="SourceCatalog", - name="src") - outputCatalog = connectionTypes.Output(doc="Aperture measurements", - dimensions=("visit", "detector", - "band"), - storageClass="SourceCatalog", - name="customAperture") + class ApertureTaskConnections( + pipeBase.PipelineTaskConnections, dimensions=("visit", "detector", "band") + ): + exposure = connectionTypes.Input( + doc="Input exposure to make measurements on", + dimensions=("visit", "detector"), + storageClass="ExposureF", + name="calexp", + ) + inputCatalog = connectionTypes.Input( + doc="Input catalog with existing measurements", + dimensions=( + "visit", + "detector", + "band", + ), + storageClass="SourceCatalog", + name="src", + ) + outputCatalog = connectionTypes.Output( + doc="Aperture measurements", + dimensions=("visit", "detector", "band"), + storageClass="SourceCatalog", + name="customAperture", + ) So what is going on here? The first thing that happens is that ApertureTaskConnections inherits from PipelineTaskConnections, that is fairly @@ -155,8 +166,9 @@ the connections class in as a parameter to the configuration class declaration. .. code-block:: python - class ApertureTaskConfig(pipeBase.PipelineTaskConfig, - pipelineConnections=ApertureTaskConnections): + class ApertureTaskConfig( + pipeBase.PipelineTaskConfig, pipelineConnections=ApertureTaskConnections + ): ... That's it. All the rest of the Config class stays the same. Below, examples @@ -166,12 +178,12 @@ look at what changes when you turn a Task into a PipelineTask. .. code-block:: python class ApertureTask(pipeBase.PipelineTask): - ... - def run(self, exposure: afwImage.Exposure, - inputCatalog: afwTable.SourceCatalog) -> pipeBase.Struct: - ... + def run( + self, exposure: afwImage.Exposure, inputCatalog: afwTable.SourceCatalog + ) -> pipeBase.Struct: + ... return pipeBase.Struct(outputCatalog=self.outputCatalog) In a simple PipelineTask like this, these are all the changes that need to be @@ -208,31 +220,32 @@ is executed. Take a look what the connection class looks like. .. code-block:: python - class ApertureTaskConnections(pipeBase.PipelineTaskConnections, - dimensions=("visit", "detector", "band")): - inputSchema = connectionTypes.InitInput(doc="Schema associated with a src catalog", - storageClass="SourceCatalog", - name="src_schema") + class ApertureTaskConnections( + pipeBase.PipelineTaskConnections, dimensions=("visit", "detector", "band") + ): + inputSchema = connectionTypes.InitInput( + doc="Schema associated with a src catalog", + storageClass="SourceCatalog", + name="src_schema", + ) ... - class ApertureTask(pipeBase.PipelineTask): + class ApertureTask(pipeBase.PipelineTask): ... - def __init__(self, config: pexConfig.Config, initInput: Mapping, - *args, **kwargs): + def __init__(self, config: pexConfig.Config, initInput: Mapping, *args, **kwargs): ... - inputSchema = initInput['inputSchema'].schema + inputSchema = initInput["inputSchema"].schema # Create a camera mapper to create a copy of the input schema self.mapper = afwTable.SchemaMapper(inputSchema) self.mapper.addMinimalSchema(inputSchema, True) # Add the new field - self.apKey = self.mapper.editOutputSchema().addField("apFlux", - type=np.float64, - doc="Ap flux - "measured") + self.apKey = self.mapper.editOutputSchema().addField( + "apFlux", type=np.float64, doc="Ap flux measured" + ) # Get the output schema self.schema = self.mapper.getOutputSchema() @@ -240,15 +253,15 @@ is executed. Take a look what the connection class looks like. # create the catalog in which new measurements will be stored self.outputCatalog = afwTable.SourceCatalog(self.schema) - def run(self, exposure: afwImage.Exposure, - inputCatalog: afwTable.SourceCatalog - ) -> pipeBase.Struct: + def run( + self, exposure: afwImage.Exposure, inputCatalog: afwTable.SourceCatalog + ) -> pipeBase.Struct: # Add in all the records from the input catalog into what will be the # output catalog self.outputCatalog.extend(inputCatalog, mapper=self.mapper) # set dimension cutouts to 3 times the apRad times 2 (for diameter) - dimensions = (3*self.apRad*2, 3*self.apRad*2) + dimensions = (3 * self.apRad * 2, 3 * self.apRad * 2) # Get indexes for each pixel indy, indx = np.indices(dimensions) @@ -259,11 +272,14 @@ is executed. Take a look what the connection class looks like. center = Point2I(source.getCentroid()) center = (center.getY(), center.getX()) # Create a cutout - stamp = exposure.image.array[center[0]-3*self.apRad: center[0]+3*self.apRad, - center[1]-3*self.apRad: center[1]+3*self.apRad] - mask = ((indy - center[0])**2 - + (indx - center[0])**2)**0.5 < self.apRad - flux = np.sum(stamp*mask) + stamp = exposure.image.array[ + center[0] - 3 * self.apRad : center[0] + 3 * self.apRad, + center[1] - 3 * self.apRad : center[1] + 3 * self.apRad, + ] + mask = ( + (indy - center[0]) ** 2 + (indx - center[0]) ** 2 + ) ** 0.5 < self.apRad + flux = np.sum(stamp * mask) # Set the flux field of this source source.set(self.apKey, flux) @@ -291,23 +307,23 @@ class. .. code-block:: python - class ApertureTaskConnections(pipeBase.PipelineTaskConnections, - dimensions=("visit", "detector", - "band", "skymap")): + class ApertureTaskConnections( + pipeBase.PipelineTaskConnections, dimensions=("visit", "detector", "band", "skymap") + ): ... - outputSchema = connectionTypes.InitOutput(doc="Schema created in Aperture PipelineTask", - storageClass="SourceCatalog", - name="customAperture_schema") + outputSchema = connectionTypes.InitOutput( + doc="Schema created in Aperture PipelineTask", + storageClass="SourceCatalog", + name="customAperture_schema", + ) ... class ApertureTask(pipeBase.PipelineTask): - ... - def __init__(self, config: pexConfig.Config, initInput:Mapping, - *args, **kwargs): + def __init__(self, config: pexConfig.Config, initInput: Mapping, *args, **kwargs): ... # Get the output schema self.schema = mapper.getOutputSchema() @@ -336,13 +352,16 @@ datasets. .. code-block:: python - class ApertureTaskConnections(pipeBase.PipelineTaskConnections, - dimensions=("visit", "detector", "band")): + class ApertureTaskConnections( + pipeBase.PipelineTaskConnections, dimensions=("visit", "detector", "band") + ): ... - background = connectionTypes.Input(doc="Background model for the exposure", - storageClass="Background", - name="calexpBackground", - dimensions=("visit", "detector", "band")) + background = connectionTypes.Input( + doc="Background model for the exposure", + storageClass="Background", + name="calexpBackground", + dimensions=("visit", "detector", "band"), + ) ... Now your `PipelineTask` will load the background each time the task is run. @@ -351,12 +370,15 @@ The simplest approach is to add a configuration field in your config class to al .. code-block:: python - class ApertureTaskConfig(pipeBase.PipelineTaskConfig, - pipelineConnections=ApertureTaskConnections): + class ApertureTaskConfig( + pipeBase.PipelineTaskConfig, pipelineConnections=ApertureTaskConnections + ): ... - doLocalBackground = pexConfig.Field(doc="Should the background be added " - "before doing photometry", - dtype=bool, default=False) + doLocalBackground = pexConfig.Field( + doc="Should the background be added before doing photometry", + dtype=bool, + default=False, + ) The ``__init__`` method of the connection class is given an instance of the task's configuration class after all overrides have been applied. This provides @@ -365,15 +387,16 @@ various configuration options. .. code-block:: python - class ApertureTaskConnections(pipeBase.PipelineTaskConnections, - dimensions=("visit", "detector", - "band", "skymap")): + class ApertureTaskConnections( + pipeBase.PipelineTaskConnections, dimensions=("visit", "detector", "band", "skymap") + ): ... - background = ct.Input(doc="Background model for the exposure", - storageClass="Background", - name="calexpBackground", - dimensions=("visit", "detector", "band", - "skymap")) + background = ct.Input( + doc="Background model for the exposure", + storageClass="Background", + name="calexpBackground", + dimensions=("visit", "detector", "band", "skymap"), + ) ... def __init__(self, *, config=None): @@ -411,14 +434,16 @@ take into account that a background may or may not be supplied. ... - class ApertureTask(pipeBase.PipelineTask): + class ApertureTask(pipeBase.PipelineTask): ... - def run(self, exposure: afwImage.Exposure, - inputCatalog: afwTable.SourceCatalog, - background: typing.Optional[afwMath.BackgroundList] = None - ) -> pipeBase.Struct: + def run( + self, + exposure: afwImage.Exposure, + inputCatalog: afwTable.SourceCatalog, + background: typing.Optional[afwMath.BackgroundList] = None, + ) -> pipeBase.Struct: # If a background is supplied, add it back to the image so local # background subtraction can be done. if background is not None: @@ -428,20 +453,18 @@ take into account that a background may or may not be supplied. # Loop over each record in the catalog for source in outputCatalog: - ... - distance = ((indy - center[0])**2 - + (indx - center[0])**2)**0.5 + distance = ((indy - center[0]) ** 2 + (indx - center[0]) ** 2) ** 0.5 mask = distance < self.apRad - flux = np.sum(stamp*mask) + flux = np.sum(stamp * mask) # Do local background subtraction if background is not None: - outerAn = distance < 2.5*self.apRad - innerAn = distance < 1.5*self.apRad + outerAn = distance < 2.5 * self.apRad + innerAn = distance < 1.5 * self.apRad annulus = outerAn - innerAn - localBackground = np.mean(exposure.image.array*annulus) - flux -= np.sum(mask)*localBackground + localBackground = np.mean(exposure.image.array * annulus) + flux -= np.sum(mask) * localBackground ... @@ -504,20 +527,25 @@ providing a way to template dataset type names. .. code-block:: python - class ApertureTaskConnections(pipeBase.PipelineTaskConnections, - defaultTemplates={"outputName": "customAperture"}, - dimensions=("visit", "detector", "band")): + class ApertureTaskConnections( + pipeBase.PipelineTaskConnections, + defaultTemplates={"outputName": "customAperture"}, + dimensions=("visit", "detector", "band"), + ): ... - - outputCatalog = connectionTypes.Output(doc="Aperture measurements", - dimensions=("visit", "detector", "band"), - storageClass="SourceCatalog", - name="{outputName}") + outputCatalog = connectionTypes.Output( + doc="Aperture measurements", + dimensions=("visit", "detector", "band"), + storageClass="SourceCatalog", + name="{outputName}", + ) ... - outputSchema = connectionTypes.InitOutput(doc="Schema created in Aperture PipelineTask", - storageClass="SourceCatalog", - name="{outputName}_schema") + outputSchema = connectionTypes.InitOutput( + doc="Schema created in Aperture PipelineTask", + storageClass="SourceCatalog", + name="{outputName}_schema", + ) In the modified connection class, the ``outputSchema`` and ``outputCatalog`` @@ -680,14 +708,15 @@ code this behavior, but are done to demonstrate how to make use of the .. code-block:: python class ApertureTask(pipeBase.PipelineTask): - ... - def run(self, exposures: List[afwImage.Exposure], - inputCatalogs: List[afwTable.SourceCatalog], - areaMasks: List[afwImage.Mask], - backgrounds: Optional[typing.List[afwMath.BackgroundList]] = None - ) -> pipeBase.Struct: + def run( + self, + exposures: List[afwImage.Exposure], + inputCatalogs: List[afwTable.SourceCatalog], + areaMasks: List[afwImage.Mask], + backgrounds: Optional[typing.List[afwMath.BackgroundList]] = None, + ) -> pipeBase.Struct: # Track the length of each catalog as to know which exposure to use # in later processing cumulativeLength = 0 @@ -696,7 +725,7 @@ code this behavior, but are done to demonstrate how to make use of the # Add in all the input catalogs into the output catalog for inCat in inputCatalogs: self.outputCatalog.extend(inCat, mapper=self.mapper) - lengths.append(len(inCat)+cumulativeLength) + lengths.append(len(inCat) + cumulativeLength) cumulativeLength += len(inCat) ... @@ -818,11 +847,14 @@ Take a look at how the ``run`` method changes to make use of this. class ApertureTask(pipeBase.PipelineTask): ... - def run(self, exposures: List[afwImage.Exposure], - inputCatalogs: List[afwTable.SourceCatalog], - areaMasks: List[afwImage.Mask], - backgrounds: Optional[typing.List[afwMath.BackgroundList]] = None - ) -> pipeBase.Struct: + + def run( + self, + exposures: List[afwImage.Exposure], + inputCatalogs: List[afwTable.SourceCatalog], + areaMasks: List[afwImage.Mask], + backgrounds: Optional[typing.List[afwMath.BackgroundList]] = None, + ) -> pipeBase.Struct: ... # track which image is being used @@ -872,9 +904,11 @@ That's exactly what should happens with ``ApertureTask`` now; it should produce .. code-block:: python - class ApertureTaskConnections(pipeBase.PipelineTaskConnections, - defaultTemplates={"outputName":"customAperture"}, - dimensions=("visit", "detector", "band")): + class ApertureTaskConnections( + pipeBase.PipelineTaskConnections, + defaultTemplates={"outputName": "customAperture"}, + dimensions=("visit", "detector", "band"), + ): ... def adjustQuantum(self, inputs, outputs, label, data_id): @@ -882,14 +916,11 @@ That's exactly what should happens with ``ApertureTask`` now; it should produce input_names = ("exposures", "inputCatalogs", "backgrounds") inputs_by_data_id = [] for name in input_names: - inputs_by_data_id.append( - {ref.dataId: ref for ref in inputs[name][1]} - ) + inputs_by_data_id.append({ref.dataId: ref for ref in inputs[name][1]}) # Intersection looks messy because dict_keys only supports |. # not an "intersection" method. data_ids_to_keep = functools.reduce( - operator.__and__, - (d.keys() for d in inputs_by_data_id) + operator.__and__, (d.keys() for d in inputs_by_data_id) ) # Pull out just the DatasetRefs that are in common in the inputs # and order them consistently (note that consistent ordering is not @@ -904,14 +935,11 @@ That's exactly what should happens with ``ApertureTask`` now; it should produce # super() later. inputs[name] = adjusted_inputs[name] # Do the same for the outputs. - outputs_by_data_id = { - ref.dataId: ref for ref in outputs["outputCatalogs"][1] - } + outputs_by_data_id = {ref.dataId: ref for ref in outputs["outputCatalogs"][1]} adjusted_outputs = { "outputCatalogs": ( outputs["outputCatalogs"][0], - [outputs_by_data_id[data_id] - for data_id in data_ids_to_keep] + [outputs_by_data_id[data_id] for data_id in data_ids_to_keep], ) } outputs["outputCatalogs"] = adjusted_outputs["outputCatalogs"] @@ -945,9 +973,12 @@ it. Take a look at how ``runQuantum`` is defined in ``PipelineTask``. class PipelineTask(Task): ... - def runQuantum(self, butlerQC: QuantumContext, - inputRefs: InputQuantizedConnection, - outputRefs: OutputQuantizedConnection): + def runQuantum( + self, + butlerQC: QuantumContext, + inputRefs: InputQuantizedConnection, + outputRefs: OutputQuantizedConnection, + ): inputs = butlerQC.get(inputRefs) outputs = self.run(**inputs) butlerQC.put(outputs, outputRefs) @@ -991,9 +1022,12 @@ with that single `lsst.daf.butler.DatasetRef`. class ApertureTask(pipeBase.PipelineTask): ... - def runQuantum(self, butlerQC: pipeBase.QuantumContext, - inputRefs: pipeBase.InputQuantizedConnection, - outputRefs: pipeBase.OutputQuantizedConnection): + def runQuantum( + self, + butlerQC: pipeBase.QuantumContext, + inputRefs: pipeBase.InputQuantizedConnection, + outputRefs: pipeBase.OutputQuantizedConnection, + ): inputs = {} for name, refs in inputRefs: inputs[name] = butlerQC.get(refs) @@ -1017,9 +1051,13 @@ demoing the concepts here. ... - def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext, - inputRefs: pipeBase.InputQuantizedConnection, - outputRefs: pipeBase.OutputQuantizedConnection): + + def runQuantum( + self, + butlerQC: pipeBase.ButlerQuantumContext, + inputRefs: pipeBase.InputQuantizedConnection, + outputRefs: pipeBase.OutputQuantizedConnection, + ): inputs = {} for name, refs in inputRefs: inputs[name] = butlerQC.get(refs) @@ -1028,26 +1066,29 @@ demoing the concepts here. lengths = [] # Remove the input catalogs from the list of inputs to the run method - inputCatalogs = inputs.pop('inputCatalogs') + inputCatalogs = inputs.pop("inputCatalogs") # Add in all the input catalogs into the output catalog cumulativeLength = 0 for inCatHandle in inputCatalogs: inCat = inCatHandle.get() - lengths.append(len(inCat)+cumulativeLength) + lengths.append(len(inCat) + cumulativeLength) cumulativeLength += len(inCat) self.outputCatalog.extend(inCat, mapper=self.mapper) # Add the catalog lengths to the inputs to the run method - inputs['lengths'] = lengths + inputs["lengths"] = lengths output = self.run(**inputs) butlerQC.put(output, outputRefs.OutputCatalog) - def run(self, exposures: List[afwImage.Exposure], - lengths: List[int], - areaMasks: List[afwImage.Mask], - backgrounds: Union[None, typing.List[afwMath.BackgroundList]] = None - ) -> pipeBase.Struct: + + def run( + self, + exposures: List[afwImage.Exposure], + lengths: List[int], + areaMasks: List[afwImage.Mask], + backgrounds: Union[None, typing.List[afwMath.BackgroundList]] = None, + ) -> pipeBase.Struct: ... Adding additional logic to a `PipelineTask` in this way is very powerful, but diff --git a/doc/lsst.pipe.base/creating-a-task.rst b/doc/lsst.pipe.base/creating-a-task.rst index 23178ebc2..8de86d068 100644 --- a/doc/lsst.pipe.base/creating-a-task.rst +++ b/doc/lsst.pipe.base/creating-a-task.rst @@ -61,8 +61,8 @@ Some important details of configurations: .. code-block:: python class ExampleSigmaClippedStatsConfig(pexConfig.Config): - """Configuration for ExampleSigmaClippedStatsTask. - """ + """Configuration for ExampleSigmaClippedStatsTask.""" + badMaskPlanes = pexConfig.ListField( dtype=str, doc="Mask planes that, if set, indicate the associated pixel should " @@ -87,8 +87,8 @@ Here is the config for ``ExampleTask``, a task that calls one subtask named ``st .. code-block:: python class ExampleConfig(pexConfig.Config): - """Configuration for ExampleTask. - """ + """Configuration for ExampleTask.""" + stats = pexConfig.ConfigurableField( doc="Subtask to compute statistics of an image", target=ExampleSigmaClippedStatsTask, @@ -227,10 +227,11 @@ For example, to look for a debug variable named "display": .. code-block:: python import lsstDebug + display = lsstDebug.Info(__name__).display if display: - # ... - pass + # ... + pass .. FIXME lsstDebug comes from ``base`` but that is not a dependency of .. this package. Linking to the base documentation is therefore problematic. @@ -288,6 +289,7 @@ There are advantages to each: .. code-block:: python from ... import FooTask + config.configurableSubtask.retarget(FooTask) - Variants subtasks are kept together in one registry, making it easier to find them. diff --git a/doc/lsst.pipe.base/task-retargeting-howto.rst b/doc/lsst.pipe.base/task-retargeting-howto.rst index e8816a16c..93fc2f610 100644 --- a/doc/lsst.pipe.base/task-retargeting-howto.rst +++ b/doc/lsst.pipe.base/task-retargeting-howto.rst @@ -35,6 +35,7 @@ For example, to retarget the subtask named ``configurableSubtask`` with a class .. code-block:: python from ... import FooTask + config.configurableSubtask.retarget(FooTask) .. TODO make this a realistic example. @@ -62,6 +63,7 @@ Here is an example that assumes a task ``FooTask`` is defined in module :file:`. .. code-block:: python import .../foo.py + config.registrySubtask.name = "foo" Besides retargeting the registry subtask, there are two ways to configure parameters for tasks in a registry: diff --git a/doc/lsst.pipe.base/testing-a-pipeline-task.rst b/doc/lsst.pipe.base/testing-a-pipeline-task.rst index fb62d675c..ad6460d9c 100644 --- a/doc/lsst.pipe.base/testing-a-pipeline-task.rst +++ b/doc/lsst.pipe.base/testing-a-pipeline-task.rst @@ -60,11 +60,11 @@ If you do need `~lsst.pipe.base.PipelineTask.runQuantum` to call `~lsst.pipe.bas butlerTests.addDataIdValue(repo, "visit", 102) butlerTests.addDataIdValue(repo, "detector", 42) butlerTests.addDatasetType( - repo, "InputType", {"instrument", "visit", "detector"}, - "ExposureF") + repo, "InputType", {"instrument", "visit", "detector"}, "ExposureF" + ) butlerTests.addDatasetType( - repo, "OutputType", {"instrument", "visit", "detector"}, - "ExposureF") + repo, "OutputType", {"instrument", "visit", "detector"}, "ExposureF" + ) ... @@ -73,8 +73,8 @@ If you do need `~lsst.pipe.base.PipelineTask.runQuantum` to call `~lsst.pipe.bas butler = butlerTests.makeTestCollection(repo) task = AwesomeTask() quantum = testUtils.makeQuantum( - task, butler, dataId, - {key: dataId for key in {"input", "output"}}) + task, butler, dataId, {key: dataId for key in {"input", "output"}} + ) run = testUtils.runTestQuantum(task, butler, quantum) # Actual input dataset omitted for simplicity run.assert_called_once() @@ -90,25 +90,26 @@ The `lsst.pipe.base.testUtils.assertValidOutput` function takes a task object an Currently, it tests for missing fields and mixing up vector and scalar values; more tests may be added in the future. .. code-block:: py - :emphasize-lines: 29-31 + :emphasize-lines: 30-32 import lsst.daf.butler.tests as butlerTests - from lsst.pipe.base import connectionTypes, PipelineTask, \ - PipelineTaskConnections + from lsst.pipe.base import connectionTypes, PipelineTask, PipelineTaskConnections from lsst.pipe.base import testUtils class MyConnections( - PipelineTaskConnections, - dimensions=("instrument", "visit", "detector")): + PipelineTaskConnections, dimensions=("instrument", "visit", "detector") + ): image = connectionTypes.Output( name="calexp", storageClass="ExposureF", - dimensions=("instrument", "visit", "detector")) + dimensions=("instrument", "visit", "detector"), + ) catalog = connectionTypes.Output( name="src", storageClass="SourceCatalog", - dimensions=("instrument", "visit", "detector")) + dimensions=("instrument", "visit", "detector"), + ) class MyTask(PipelineTask): @@ -135,25 +136,23 @@ The `lsst.pipe.base.testUtils.assertValidInitOutput` function takes a task objec The tests are analogous to :ref:`those for assertValidOutput `. .. code-block:: py - :emphasize-lines: 28-29 + :emphasize-lines: 26-27 import lsst.afw.table as afwTable import lsst.daf.butler.tests as butlerTests - from lsst.pipe.base import connectionTypes, PipelineTask, \ - PipelineTaskConnections + from lsst.pipe.base import connectionTypes, PipelineTask, PipelineTaskConnections from lsst.pipe.base import testUtils class MyConnections( - PipelineTaskConnections, - dimensions=("instrument", "visit", "detector")): - schema = connectionTypes.InitOutput( - name="srcSchema", - storageClass="SourceCatalog") + PipelineTaskConnections, dimensions=("instrument", "visit", "detector") + ): + schema = connectionTypes.InitOutput(name="srcSchema", storageClass="SourceCatalog") catalog = connectionTypes.Output( name="src", storageClass="SourceCatalog", - dimensions=("instrument", "visit", "detector")) + dimensions=("instrument", "visit", "detector"), + ) class MyTask(PipelineTask): @@ -182,26 +181,34 @@ Optional init-inputs can be tested by calling `lsst.pipe.base.testUtils.getInitI There is currently no test framework for the use of init-inputs in task constructors. .. code-block:: py - :emphasize-lines: 42-43, 49-50 + :emphasize-lines: 49-50, 56-57 import lsst.daf.butler.tests as butlerTests - from lsst.pipe.base import connectionTypes, PipelineTask, \ - PipelineTaskConnections, PipelineTaskConfig + from lsst.pipe.base import ( + connectionTypes, + PipelineTask, + PipelineTaskConnections, + PipelineTaskConfig, + ) from lsst.pipe.base import testUtils # A task that can take an Exposure xor a Catalog # Don't try this at home! - class OrConnections(PipelineTaskConnections, - dimensions=("instrument", "visit", "detector")): + + class OrConnections( + PipelineTaskConnections, dimensions=("instrument", "visit", "detector") + ): exp = connectionTypes.Input( name="calexp", storageClass="ExposureF", - dimensions=("instrument", "visit", "detector")) + dimensions=("instrument", "visit", "detector"), + ) cat = connectionTypes.Input( name="src", storageClass="SourceCatalog", - dimensions=("instrument", "visit", "detector")) + dimensions=("instrument", "visit", "detector"), + ) def __init__(self, *, config=None): super().__init__(config=config) @@ -211,8 +218,7 @@ There is currently no test framework for the use of init-inputs in task construc self.inputs.remove("cat") - class OrConfig(PipelineTaskConfig, - pipelineConnections=OrConnections): + class OrConfig(PipelineTaskConfig, pipelineConnections=OrConnections): doCatalog = Field(dtype=bool, default=False) @@ -248,15 +254,18 @@ All tests done by `lintConnections` are heuristic, looking for common patterns o Advanced users who are *deliberately* bending the usual rules can use keywords to turn off specific tests. .. code-block:: py - :emphasize-lines: 9-10 + :emphasize-lines: 12-13 - class ListConnections(PipelineTaskConnections, - dimensions=("instrument", "visit", "detector")): + class ListConnections( + PipelineTaskConnections, dimensions=("instrument", "visit", "detector") + ): cat = connectionTypes.Input( name="src", storageClass="SourceCatalog", dimensions=("instrument", "visit", "detector"), - multiple=True) # force a list of one catalog + multiple=True, # force a list of one catalog + ) + lintConnections(ListConnections) # warns that cat always has one input lintConnections(ListConnections, checkUnnecessaryMultiple=False) # passes From b28e971375b34f0e791a3a1641ba2be2fa2720c8 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Mon, 10 Jul 2023 13:35:35 -0700 Subject: [PATCH 4/7] Remove the --- from the yaml file --- .github/workflows/rebase_checker.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/rebase_checker.yaml b/.github/workflows/rebase_checker.yaml index 62aeca77e..65516d920 100644 --- a/.github/workflows/rebase_checker.yaml +++ b/.github/workflows/rebase_checker.yaml @@ -1,4 +1,3 @@ ---- name: Check that 'main' is not merged into the development branch on: pull_request From 4e850338fa9e05e7d017fc147eddcfc7ef1060bb Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Mon, 10 Jul 2023 13:41:20 -0700 Subject: [PATCH 5/7] Use modern type annotations in worked examples --- doc/lsst.pipe.base/creating-a-pipelinetask.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/lsst.pipe.base/creating-a-pipelinetask.rst b/doc/lsst.pipe.base/creating-a-pipelinetask.rst index 90e36f4d9..6d89e7ed4 100644 --- a/doc/lsst.pipe.base/creating-a-pipelinetask.rst +++ b/doc/lsst.pipe.base/creating-a-pipelinetask.rst @@ -428,8 +428,6 @@ take into account that a background may or may not be supplied. .. code-block:: python ... - import typing - import lsst.afw.math as afwMath ... @@ -442,7 +440,7 @@ take into account that a background may or may not be supplied. self, exposure: afwImage.Exposure, inputCatalog: afwTable.SourceCatalog, - background: typing.Optional[afwMath.BackgroundList] = None, + background: afwMath.BackgroundList | None = None, ) -> pipeBase.Struct: # If a background is supplied, add it back to the image so local # background subtraction can be done. @@ -715,7 +713,7 @@ code this behavior, but are done to demonstrate how to make use of the exposures: List[afwImage.Exposure], inputCatalogs: List[afwTable.SourceCatalog], areaMasks: List[afwImage.Mask], - backgrounds: Optional[typing.List[afwMath.BackgroundList]] = None, + backgrounds: list[afwMath.BackgroundList] | None = None, ) -> pipeBase.Struct: # Track the length of each catalog as to know which exposure to use # in later processing @@ -853,7 +851,7 @@ Take a look at how the ``run`` method changes to make use of this. exposures: List[afwImage.Exposure], inputCatalogs: List[afwTable.SourceCatalog], areaMasks: List[afwImage.Mask], - backgrounds: Optional[typing.List[afwMath.BackgroundList]] = None, + backgrounds: list[afwMath.BackgroundList] | None = None, ) -> pipeBase.Struct: ... @@ -1087,7 +1085,7 @@ demoing the concepts here. exposures: List[afwImage.Exposure], lengths: List[int], areaMasks: List[afwImage.Mask], - backgrounds: Union[None, typing.List[afwMath.BackgroundList]] = None, + backgrounds: list[afwMath.BackgroundList] | None = None, ) -> pipeBase.Struct: ... From e42e159d3fb1a370768aee343a8fe85a18ceba81 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Mon, 10 Jul 2023 13:42:18 -0700 Subject: [PATCH 6/7] Fix some unnecessary reformatting from black --- doc/lsst.pipe.base/creating-a-pipelinetask.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/lsst.pipe.base/creating-a-pipelinetask.rst b/doc/lsst.pipe.base/creating-a-pipelinetask.rst index 6d89e7ed4..a9b5910c6 100644 --- a/doc/lsst.pipe.base/creating-a-pipelinetask.rst +++ b/doc/lsst.pipe.base/creating-a-pipelinetask.rst @@ -118,11 +118,7 @@ task). ) inputCatalog = connectionTypes.Input( doc="Input catalog with existing measurements", - dimensions=( - "visit", - "detector", - "band", - ), + dimensions=("visit", "detector", "band"), storageClass="SourceCatalog", name="src", ) From e705081a641257fa9750a477fff8a5ec54beec54 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Mon, 10 Jul 2023 10:11:19 -0700 Subject: [PATCH 7/7] Add action to check for DO NOT MERGE commits * Checks summary for DO NOT MERGE * Checks requirements.txt for ticket branches --- .github/workflows/do_not_merge.yaml | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/do_not_merge.yaml diff --git a/.github/workflows/do_not_merge.yaml b/.github/workflows/do_not_merge.yaml new file mode 100644 index 000000000..d602c38c9 --- /dev/null +++ b/.github/workflows/do_not_merge.yaml @@ -0,0 +1,41 @@ +name: "Check commits can be merged" +on: + push: + branches: + - main + pull_request: + +jobs: + do-not-merge-checker: + runs-on: ubuntu-latest + + steps: + - name: Check that there are no commits that should not be merged + uses: gsactions/commit-message-checker@v2 + with: + excludeDescription: "true" # optional: this excludes the description body of a pull request + excludeTitle: "true" # optional: this excludes the title of a pull request + checkAllCommitMessages: "true" # optional: this checks all commits associated with a pull request + accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true + # Check for message indicating that there is a commit that should + # not be merged. + pattern: ^(?!DO NOT MERGE) + flags: "i" + error: | + "This step failed because there is a commit containing the text + 'DO NOT MERGE'. Remove this commit from the branch before merging + or change the commit summary." + + - uses: actions/checkout@v3 + + - name: Check requirements.txt for branches + shell: bash + run: | + FILE=requirements.txt + MATCH=tickets/DM- + if grep -q $MATCH $FILE + then + echo "Ticket branches found in $FILE:" + grep -n $MATCH $FILE + exit 1 + fi