Merge pull request #211 from lsst/tickets/DM-41530

DM-41530: Add individual flag fields to APDB schema

data/flag-rename-rules.yaml

@@ -5,3 +5,4 @@ flag_rename_rules:
     - ['slot_Centroid', 'centroid']
     - ['slot_PsfFlux', 'psfFlux']
     - ['slot_Shape', 'shape']
+    - ['ext_trailedSources_Naive', 'trail']

python/lsst/ap/association/diaForcedSource.py

@@ -235,7 +235,6 @@ def _calibrate_and_merge(self,
         output_catalog["band"] = diff_exp.getFilter().bandLabel
         output_catalog["time_processed"] = DateTime.now().toPython()
         # TODO: propagate actual flags (DM-42355)
-        output_catalog["flags"] = 0
 
         # Drop superfluous columns from output DataFrame.
         output_catalog.drop(columns=self.config.dropColumns, inplace=True)

python/lsst/ap/association/diaPipe.py

@@ -321,7 +321,6 @@ def setDefaults(self):
         self.apdb.dia_object_columns = []
         self.diaCalculation.plugins = ["ap_meanPosition",
                                        "ap_nDiaSources",
-                                       "ap_diaObjectFlag",
                                        "ap_meanFlux",
                                        "ap_percentileFlux",
                                        "ap_sigmaFlux",
@@ -564,7 +563,7 @@ def run(self,
             # alertPackager needs correct columns
             diaForcedSources = pd.DataFrame(columns=[
                 "diaForcedSourceId", "diaObjectID", "ccdVisitID", "psfFlux", "psfFluxErr",
-                "x", "y", "flags", "midpointMjdTai", "band",
+                "x", "y", "midpointMjdTai", "band",
             ])
 
         # Store DiaSources, updated DiaObjects, and DiaForcedSources in the
@@ -695,8 +694,7 @@ def _initialize_dia_object(self, objId):
                           "pmParallaxNdata": 0,
                           "nearbyObj1": 0,
                           "nearbyObj2": 0,
-                          "nearbyObj3": 0,
-                          "flags": 0}
+                          "nearbyObj3": 0}
         for f in ["u", "g", "r", "i", "z", "y"]:
             new_dia_object["%s_psfFluxNdata" % f] = 0
         return pd.Series(data=new_dia_object)

python/lsst/ap/association/trailedSourceFilter.py

@@ -85,17 +85,21 @@ def run(self, dia_sources, exposure_time):
             (`pandas.DataFrame`)
         """
 
-        flag_map = os.path.join(utils.getPackageDir("ap_association"), "data/association-flag-map.yaml")
-        unpacker = UnpackApdbFlags(flag_map, "DiaSource")
-        flags = unpacker.unpack(dia_sources["flags"], "flags")
+        if "flags" in dia_sources.columns:
+            flag_map = os.path.join(utils.getPackageDir("ap_association"), "data/association-flag-map.yaml")
+            unpacker = UnpackApdbFlags(flag_map, "DiaSource")
+            flags = unpacker.unpack(dia_sources["flags"], "flags")
+            trail_edge_flags = flags["ext_trailedSources_Naive_flag_edge"]
+        else:
+            trail_edge_flags = dia_sources["trail_flag_edge"]
 
-        trail_mask = self._check_dia_source_trail(dia_sources, exposure_time, flags)
+        trail_mask = self._check_dia_source_trail(dia_sources, exposure_time, trail_edge_flags)
 
         return pipeBase.Struct(
             diaSources=dia_sources[~trail_mask].reset_index(drop=True),
             longTrailedDiaSources=dia_sources[trail_mask].reset_index(drop=True))
 
-    def _check_dia_source_trail(self, dia_sources, exposure_time, flags):
+    def _check_dia_source_trail(self, dia_sources, exposure_time, trail_edge_flags):
         """Find DiaSources that have long trails.
 
         Return a mask of sources with lengths greater than
@@ -108,18 +112,18 @@ def _check_dia_source_trail(self, dia_sources, exposure_time, flags):
             Input DIASources to check for trail lengths.
         exposure_time : `float`
             Exposure time from difference image.
-        flags : 'numpy.ndArray'
-            Boolean array of flags from the DIASources.
+        trail_edge_flags : 'numpy.ndArray'
+            Boolean array of trail_flag_edge flags from the DIASources.
 
         Returns
         -------
         trail_mask : `pandas.DataFrame`
             Boolean mask for DIASources which are greater than the
-            cutoff length and have the edge flag set.
+            cutoff length or have the edge flag set.
         """
         trail_mask = (dia_sources.loc[:, "trailLength"].values[:]
                       >= (self.config.max_trail_length*exposure_time))
 
-        trail_mask[np.where(flags['ext_trailedSources_Naive_flag_edge'])] = True
+        trail_mask[np.where(trail_edge_flags)] = True
 
         return trail_mask

python/lsst/ap/association/transformDiaSourceCatalog.py

@@ -99,11 +99,13 @@ class TransformDiaSourceCatalogConfig(TransformCatalogBaseConfig,
         doc="Input DiaSource catalog contains SkySources that should be "
             "removed before storing the output DiaSource catalog."
     )
+    # TODO: remove on DM-41532
     doPackFlags = pexConfig.Field(
         dtype=bool,
-        default=True,
+        default=False,
         doc="Do pack the flags into one integer column named 'flags'."
-            "If False, instead produce one boolean column per flag."
+            "If False, instead produce one boolean column per flag.",
+        deprecated="This field is no longer used. Will be removed after v28."
     )
     doIncludeReliability = pexConfig.Field(
         dtype=bool,

tests/test_diaForcedSource.py

@@ -173,7 +173,7 @@ def setUp(self):
         # above list of ids.
         self.expectedDiaForcedSources = 6
 
-        self.expected_n_columns = 14
+        self.expected_n_columns = 13
 
     def testRun(self):
         """Test that forced source catalogs are successfully created and have

tests/test_loadDiaCatalogs.py

@@ -81,7 +81,7 @@ def setUp(self):
 
         # These columns are not in the DPDD, yet do appear in DiaSource.yaml.
         # We don't need to check them against the default APDB schema.
-        self.ignoreColumns = ["band", "bboxSize", "isDipole"]
+        self.ignoreColumns = ["band", "bboxSize", "isDipole", "flags"]
 
     def testRun(self):
         """Test the full run method for the loader.

tests/test_packageAlerts.py

@@ -126,7 +126,6 @@ def mock_alert(alert_id):
             "snr": np.float32(6.7),
             "psfFlux": np.float32(700.0),
             "psfFluxErr": np.float32(90.0),
-            "flags": 12345,
         }
     }
 
@@ -197,10 +196,6 @@ def setUp(self):
             diaSourceHistory,
             diaForcedSources,
             self.exposure.visitInfo.date.toAstropy())
-        self.diaObjects.replace(to_replace=[None], value=np.nan, inplace=True)
-        diaSourceHistory.replace(to_replace=[None], value=np.nan, inplace=True)
-        self.diaForcedSources.replace(to_replace=[None], value=np.nan,
-                                      inplace=True)
         diaSourceHistory["programId"] = 0
 
         self.diaSources = diaSourceHistory.loc[[(1, "g", 9), (2, "g", 10)], :]

tests/test_transformDiaSourceCatalog.py

@@ -177,9 +177,11 @@ def test_computeBBoxSize(self):
             self.assertEqual(size, self.bboxSize)
         self.assertEqual(len(boxSizes), self.nSources)
 
+    # TODO: remove in DM-41532
     def test_bit_unpacker(self):
         """Test that the integer bit packer is functioning correctly.
         """
+        self.config.doPackFlags = True
         transform = TransformDiaSourceCatalogTask(initInputs=self.initInputs,
                                                   config=self.config)
         for idx, obj in enumerate(self.inputCatalog):

tests/utils_tests.py

@@ -65,7 +65,6 @@ def makeDiaObjects(nObjects, exposure, rng):
                      "nearbyObj1": 0,
                      "nearbyObj2": 0,
                      "nearbyObj3": 0,
-                     "flags": 1,
                      "nDiaSources": 5}
         for f in ["u", "g", "r", "i", "z", "y"]:
             newObject["%s_psfFluxNdata" % f] = 0
@@ -124,8 +123,7 @@ def makeDiaSources(nSources, diaObjectIds, exposure, rng, randomizeObjects=False
                      "band": exposure.getFilter().bandLabel,
                      "psfNdata": 0,
                      "trailNdata": 0,
-                     "dipoleNdata": 0,
-                     "flags": 1})
+                     "dipoleNdata": 0})
 
     return pd.DataFrame(data=data)
 
@@ -167,8 +165,7 @@ def makeDiaForcedSources(nForcedSources, diaObjectIds, exposure, rng, randomizeO
                      "diaObjectId": objId,
                      "midpointMjdTai": midpointMjdTai + 1.0 * i,
                      "time_processed": datetime.datetime.now(),
-                     "band": exposure.getFilter().bandLabel,
-                     "flags": 0})
+                     "band": exposure.getFilter().bandLabel})
 
     return pd.DataFrame(data=data)