Rework adaptive detection into a replacement for regular detection.

Author: TallJimbo

python/lsst/pipe/tasks/calibrateImage.py

@@ -286,18 +286,14 @@ class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Cali
         doc="Task to perform intial background subtraction, before first detection pass.",
     )
     psf_detection = pexConfig.ConfigurableField(
-        target=lsst.meas.algorithms.SourceDetectionTask,
+        target=AdaptiveThresholdDetectionTask,
         doc="Task to detect sources for PSF determination."
     )
     do_adaptive_threshold_detection = pexConfig.Field(
         dtype=bool,
         default=True,
         doc="Implement the adaptive detection thresholding approach?",
     )
-    psf_adaptive_threshold_detection = pexConfig.ConfigurableField(
-        target=AdaptiveThresholdDetectionTask,
-        doc="Task to adaptively detect sources for PSF determination.",
-    )
     psf_source_measurement = pexConfig.ConfigurableField(
         target=lsst.meas.base.SingleFrameMeasurementTask,
         doc="Task to measure sources to be used for psf estimation."
@@ -472,13 +468,8 @@ def setDefaults(self):
         # thresholdValue * includeThresholdMultiplier. The low thresholdValue
         # ensures that the footprints are large enough for the noise replacer
         # to mask out faint undetected neighbors that are not to be measured.
-        self.psf_detection.thresholdValue = 10.0
-        self.psf_detection.includeThresholdMultiplier = 5.0
-        # TODO investigation: Probably want False here, but that may require
-        # tweaking the background spatial scale, to make it small enough to
-        # prevent extra peaks in the wings of bright objects.
-        self.psf_detection.doTempLocalBackground = False
-        self.psf_detection.reEstimateBackground = False
+        self.psf_detection.baseline.thresholdValue = 10.0
+        self.psf_detection.baseline.includeThresholdMultiplier = 5.0
 
         # Minimal measurement plugins for PSF determination.
         # TODO DM-39203: We can drop GaussianFlux and PsfFlux, if we use
@@ -622,15 +613,15 @@ def validate(self):
                     "CalibrateImageTask.psf_subtract_background must be configured with "
                     "doApplyFlatBackgroundRatio=True if do_illumination_correction=True."
                 )
-            if self.psf_detection.reEstimateBackground:
+            if getattr(self.psf_detection, "reEstimateBackground", False):
                 if not self.psf_detection.doApplyFlatBackgroundRatio:
                     raise pexConfig.FieldValidationError(
                         CalibrateImageConfig.psf_detection,
                         self,
                         "CalibrateImageTask.psf_detection background must be configured with "
                         "doApplyFlatBackgroundRatio=True if do_illumination_correction=True."
                     )
-            if self.star_detection.reEstimateBackground:
+            if getattr(self.star_detection, "reEstimateBackground", False):
                 if not self.star_detection.doApplyFlatBackgroundRatio:
                     raise pexConfig.FieldValidationError(
                         CalibrateImageConfig.star_detection,
@@ -643,17 +634,17 @@ def validate(self):
             if not os.getenv("SATTLE_URI_BASE"):
                 raise pexConfig.FieldValidationError(CalibrateImageConfig.run_sattle, self,
                                                      "Sattle requested but URI environment variable not set.")
+        if (
+            issubclass(self.psf_detection.target, AdaptiveThresholdDetectionTask)
+            != self.do_adaptive_threshold_detection
+        ):
+            raise pexConfig.FieldValidationError(
+                CalibrateImageConfig.psf_detection, self,
+                "AdaptiveThresholdDetectionTask must be used for the psf_detection subtask "
+                "if and only if do_adaptive_threshold_detection is True."
+            )
 
         if not self.do_adaptive_threshold_detection:
-            if not self.psf_detection.reEstimateBackground:
-                raise pexConfig.FieldValidationError(
-                    CalibrateImageConfig.psf_detection,
-                    self,
-                    "If not using the adaptive threshold detection implementation (i.e. "
-                    "config.do_adaptive_threshold_detection = False), CalibrateImageTask.psf_detection "
-                    "background must be configured with "
-                    "reEstimateBackground = True to maintain original behavior."
-                )
             if not self.star_detection.reEstimateBackground:
                 raise pexConfig.FieldValidationError(
                     CalibrateImageConfig.psf_detection,
@@ -695,7 +686,6 @@ def __init__(self, initial_stars_schema=None, **kwargs):
         afwTable.CoordKey.addErrorFields(self.psf_schema)
         self.makeSubtask("psf_detection", schema=self.psf_schema)
         self.makeSubtask("psf_source_measurement", schema=self.psf_schema)
-        self.makeSubtask("psf_adaptive_threshold_detection")
         self.makeSubtask("psf_measure_psf", schema=self.psf_schema)
         self.makeSubtask("psf_normalized_calibration_flux", schema=self.psf_schema)
 
@@ -948,11 +938,12 @@ def run(
                 illumination_correction,
             )
 
-            result.psf_stars_footprints, result.background, _, adaptiveDetResStruct = self._compute_psf(
+            psf_detections, result.background, _ = self._compute_psf(
                 result.exposure,
                 id_generator,
                 background_to_photometric_ratio=result.background_to_photometric_ratio,
             )
+            result.psf_stars_footprints = psf_detections.sources
             have_fit_psf = True
 
             # Check if all centroids have been flagged. This should happen
@@ -999,7 +990,7 @@ def run(
                 result.background,
                 id_generator,
                 background_to_photometric_ratio=result.background_to_photometric_ratio,
-                adaptiveDetResStruct=adaptiveDetResStruct,
+                psf_detections=psf_detections,
                 num_astrometry_matches=num_astrometry_matches,
             )
             psf = result.exposure.getPsf()
@@ -1147,8 +1138,10 @@ def _compute_psf(self, exposure, id_generator, background_to_photometric_ratio=N
 
         Returns
         -------
-        sources : `lsst.afw.table.SourceCatalog`
-            Catalog of detected bright sources.
+        detections : `lsst.pipe.base.Struct`
+            Struct returned by the ``psf_detection`` subtask, with its
+            ``sources`` attribute further updated by measurement and PSF
+            modeling.
         background : `lsst.afw.math.BackgroundList`
             Background that was fit to the exposure during detection.
         cell_set : `lsst.afw.math.SpatialCellSet`
@@ -1191,25 +1184,12 @@ def log_psf(msg, addToMetadata=False):
         self.psf_repair.run(exposure=exposure, keepCRs=True)
 
         table = afwTable.SourceTable.make(self.psf_schema, id_generator.make_table_id_factory())
-        if not self.config.do_adaptive_threshold_detection:
-            adaptiveDetResStruct = None
-            # Re-estimate the background during this detection step, so that
-            # measurement uses the most accurate background-subtraction.
-            detections = self.psf_detection.run(
-                table=table,
-                exposure=exposure,
-                background=background,
-                backgroundToPhotometricRatio=background_to_photometric_ratio,
-            )
-        else:
-            initialThreshold = self.config.psf_detection.thresholdValue
-            initialThresholdMultiplier = self.config.psf_detection.includeThresholdMultiplier
-            adaptiveDetResStruct = self.psf_adaptive_threshold_detection.run(
-                table, exposure,
-                initialThreshold=initialThreshold,
-                initialThresholdMultiplier=initialThresholdMultiplier,
-            )
-            detections = adaptiveDetResStruct.detections
+        detections = self.psf_detection.run(
+            table=table,
+            exposure=exposure,
+            background=background,
+            backgroundToPhotometricRatio=background_to_photometric_ratio,
+        )
 
         self.metadata["initial_psf_positive_footprint_count"] = detections.numPos
         self.metadata["initial_psf_negative_footprint_count"] = detections.numNeg
@@ -1258,7 +1238,7 @@ def log_psf(msg, addToMetadata=False):
 
         # PSF is set on exposure; candidates are returned to use for
         # calibration flux normalization and aperture corrections.
-        return detections.sources, background, psf_result.cellSet, adaptiveDetResStruct
+        return detections, background, psf_result.cellSet
 
     def _measure_aperture_correction(self, exposure, bright_sources):
         """Measure and set the ApCorrMap on the Exposure, using
@@ -1290,8 +1270,8 @@ def _measure_aperture_correction(self, exposure, bright_sources):
 
         exposure.info.setApCorrMap(ap_corr_map)
 
-    def _find_stars(self, exposure, background, id_generator, background_to_photometric_ratio=None,
-                    adaptiveDetResStruct=None, num_astrometry_matches=None):
+    def _find_stars(self, exposure, background, id_generator, background_to_photometric_ratio=None, *,
+                    psf_detections, num_astrometry_matches):
         """Detect stars on an exposure that has a PSF model, and measure their
         PSF, circular aperture, compensated gaussian fluxes.
 
@@ -1307,6 +1287,10 @@ def _find_stars(self, exposure, background, id_generator, background_to_photomet
         background_to_photometric_ratio : `lsst.afw.image.Image`, optional
             Image to convert photometric-flattened image to
             background-flattened image.
+        psf_detections : `lsst.pipe.base.Struct`
+            Struct returned from the ``psf_detection`` subtask.
+        num_astrometry_matches : `int`
+            Number of astrometry matches.
 
         Returns
         -------
@@ -1321,14 +1305,14 @@ def _find_stars(self, exposure, background, id_generator, background_to_photomet
         maxAdaptiveDetIter = 8
         adaptiveDetIter = 0
         threshFactor = 0.2
-        if adaptiveDetResStruct is not None:
+        if self.config.do_adaptive_threshold_detection:
             while inAdaptiveDet and adaptiveDetIter < maxAdaptiveDetIter:
                 inAdaptiveDet = False
                 adaptiveDetectionConfig = lsst.meas.algorithms.SourceDetectionConfig()
                 adaptiveDetectionConfig.reEstimateBackground = False
                 adaptiveDetectionConfig.includeThresholdMultiplier = 2.0
                 psfThreshold = (
-                    adaptiveDetResStruct.thresholdValue*adaptiveDetResStruct.includeThresholdMultiplier
+                    psf_detections.thresholdValue*psf_detections.includeThresholdMultiplier
                 )
                 adaptiveDetectionConfig.thresholdValue = max(
                     self.config.star_detection.thresholdValue,

tests/test_calibrateImage.py

@@ -39,6 +39,7 @@
 import lsst.daf.butler.tests as butlerTests
 import lsst.geom
 import lsst.meas.algorithms
+from lsst.meas.algorithms.adaptive_thresholds import AdaptiveThresholdDetectionTask
 from lsst.meas.algorithms import testUtils
 import lsst.meas.extensions.psfex
 import lsst.meas.base
@@ -115,6 +116,14 @@ def setUp(self):
 
         # Test-specific configuration:
         self.config = CalibrateImageTask.ConfigClass()
+        # Maintain original, no adaptive threshold detection, configs values.
+        self.config.do_adaptive_threshold_detection = False
+        self.config.psf_detection.retarget(lsst.meas.algorithms.SourceDetectionTask)
+        self.config.psf_detection.thresholdValue = 10
+        self.config.psf_detection.includeThresholdMultiplier = 5
+        self.config.psf_detection.doTempLocalBackground = False
+        self.config.psf_detection.reEstimateBackground = True
+        self.config.star_detection.reEstimateBackground = True
         # We don't have many sources, so have to fit simpler models.
         self.config.psf_detection.background.approxOrderX = 1
         self.config.star_detection.background.approxOrderX = 1
@@ -127,10 +136,6 @@ def setUp(self):
         self.config.astrometry.sourceSelector["science"].flags.good = []
         self.config.astrometry.matcher.numPointsForShape = 3
         self.config.run_sattle = False
-        # Maintain original, no adaptive threshold detection, configs values.
-        self.config.do_adaptive_threshold_detection = False
-        self.config.psf_detection.reEstimateBackground = True
-        self.config.star_detection.reEstimateBackground = True
         # ApFlux has more noise than PsfFlux (the latter unrealistically small
         # in this test data), so we need to do magnitude rejection at higher
         # sigma, otherwise we can lose otherwise good sources.
@@ -241,19 +246,20 @@ def test_run(self):
 
         self._check_run(calibrate, result)
 
-    def test_run_adaptive_threshold_deteection(self):
+    def test_run_adaptive_threshold_detection(self):
         """Test that run() runs with adaptive threshold detection turned on.
         """
-        config = copy.copy(self.config)
         # Set the adaptive threshold detection, config values...
-        config.do_adaptive_threshold_detection = True
-        config.psf_adaptive_threshold_detection.minFootprint = 4
-        config.psf_adaptive_threshold_detection.minIsolated = 4
-        config.psf_adaptive_threshold_detection.sufficientIsolated = 4
-        config.psf_detection.reEstimateBackground = False
-        config.star_detection.reEstimateBackground = False
+        self.config.do_adaptive_threshold_detection = True
+        self.config.psf_detection.retarget(AdaptiveThresholdDetectionTask)
+        self.config.psf_detection.baseline.thresholdValue = 10
+        self.config.psf_detection.baseline.includeThresholdMultiplier = 5
+        self.config.psf_detection.minFootprint = 4
+        self.config.psf_detection.minIsolated = 4
+        self.config.psf_detection.sufficientIsolated = 4
+        self.config.star_detection.reEstimateBackground = False
 
-        calibrate = CalibrateImageTask(config=config)
+        calibrate = CalibrateImageTask(config=self.config)
         calibrate.astrometry.setRefObjLoader(self.ref_loader)
         calibrate.photometry.match.setRefObjLoader(self.ref_loader)
         with self.assertLogs("lsst.calibrateImage", level="INFO") as cm:
@@ -341,7 +347,8 @@ def test_compute_psf(self):
         that a PSF is assigned to the expopsure.
         """
         calibrate = CalibrateImageTask(config=self.config)
-        psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_detections, background, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_stars = psf_detections.sources
 
         # Catalog ids should be very large from this id generator.
         self.assertTrue(all(psf_stars['id'] > 1000000000))
@@ -413,7 +420,8 @@ def test_measure_aperture_correction(self):
         exposure.
         """
         calibrate = CalibrateImageTask(config=self.config)
-        psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_detections, _, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_stars = psf_detections.sources
 
         # First check that the exposure doesn't have an ApCorrMap.
         self.assertIsNone(self.exposure.apCorrMap)
@@ -428,10 +436,12 @@ def test_find_stars(self):
         in the image and returns them in the output catalog.
         """
         calibrate = CalibrateImageTask(config=self.config)
-        psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_detections, background, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_stars = psf_detections.sources
         calibrate._measure_aperture_correction(self.exposure, psf_stars)
 
-        stars = calibrate._find_stars(self.exposure, background, self.id_generator)
+        stars = calibrate._find_stars(self.exposure, background, self.id_generator,
+                                      psf_detections=psf_detections, num_astrometry_matches=0)
 
         # Catalog ids should be very large from this id generator.
         self.assertTrue(all(stars['id'] > 1000000000))
@@ -456,13 +466,13 @@ def test_astrometry(self):
         """
         calibrate = CalibrateImageTask(config=self.config)
         calibrate.astrometry.setRefObjLoader(self.ref_loader)
-        psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_detections, background, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_stars = psf_detections.sources
         calibrate._measure_aperture_correction(self.exposure, psf_stars)
         calibrate._fit_astrometry(self.exposure, psf_stars)
 
         # Check that we got reliable matches with the truth coordinates.
-        sky = psf_stars["sky_source"]
-        fitted = SkyCoord(psf_stars[~sky]['coord_ra'], psf_stars[~sky]['coord_dec'], unit="radian")
+        fitted = SkyCoord(psf_stars['coord_ra'], psf_stars['coord_dec'], unit="radian")
         truth = SkyCoord(self.truth_cat['coord_ra'], self.truth_cat['coord_dec'], unit="radian")
         idx, d2d, _ = fitted.match_to_catalog_sky(truth)
         np.testing.assert_array_less(d2d.to_value(u.milliarcsecond), 35.0)
@@ -474,9 +484,11 @@ def test_photometry(self):
         calibrate = CalibrateImageTask(config=self.config)
         calibrate.astrometry.setRefObjLoader(self.ref_loader)
         calibrate.photometry.match.setRefObjLoader(self.ref_loader)
-        psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_detections, background, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_stars = psf_detections.sources
         calibrate._measure_aperture_correction(self.exposure, psf_stars)
-        stars = calibrate._find_stars(self.exposure, background, self.id_generator)
+        stars = calibrate._find_stars(self.exposure, background, self.id_generator,
+                                      psf_detections=psf_detections, num_astrometry_matches=0)
         calibrate._fit_astrometry(self.exposure, stars)
 
         stars, matches, meta, photoCalib = calibrate._fit_photometry(self.exposure, stars)
@@ -515,9 +527,11 @@ def test_match_psf_stars(self):
         and candidates.
         """
         calibrate = CalibrateImageTask(config=self.config)
-        psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_detections, background, _ = calibrate._compute_psf(self.exposure, self.id_generator)
+        psf_stars = psf_detections.sources
         calibrate._measure_aperture_correction(self.exposure, psf_stars)
-        stars = calibrate._find_stars(self.exposure, background, self.id_generator)
+        stars = calibrate._find_stars(self.exposure, background, self.id_generator,
+                                      psf_detections=psf_detections, num_astrometry_matches=0)
 
         # There should be no psf-related flags set at first.
         self.assertEqual(stars["calib_psf_candidate"].sum(), 0)
@@ -829,7 +843,6 @@ def test_runQuantum_illumination_correction(self):
         config = CalibrateImageTask.ConfigClass()
         config.do_illumination_correction = True
         config.psf_subtract_background.doApplyFlatBackgroundRatio = True
-        config.psf_detection.doApplyFlatBackgroundRatio = True
         config.star_detection.doApplyFlatBackgroundRatio = True
         task = CalibrateImageTask(config=config)
         lsst.pipe.base.testUtils.assertValidInitOutput(task)