Merge pull request #170 from lsst/tickets/DM-37196

DM-37196: Update fields to reflect new APDB schema
lsst · Jun 5, 2023 · e97dcde · e97dcde
2 parents ec73c80 + ac6c309
commit e97dcde
Show file tree

Hide file tree

Showing 19 changed files with 328 additions and 412 deletions.
diff --git a/data/DiaSource.yaml b/data/DiaSource.yaml
@@ -7,9 +7,9 @@ funcs:
     ccdVisitId:
         functor: Column
         args: ccdVisitId
-    filterName:
+    band:
         functor: Column
-        args: filterName
+        args: band
     diaObjectId:
         functor: Column
         args: diaObjectId
@@ -19,22 +19,26 @@ funcs:
     parentDiaSourceId:
         functor: Column
         args: parent
-    midPointTai:
+    midpointMjdTai:
         functor: Column
-        args: midPointTai
+        args: midpointMjdTai
     bboxSize:
         functor: Column
         args: bboxSize
+    # TODO: is this one really necessary here?
+    time_processed:  # filled in automatically
+        functor: Column
+        args: time_processed
     flags:
         functor: Column
         args: flags
     ra:
         functor: RAColumn
     # raErr: not available yet DM-15180
-    decl:
+    dec:
         functor: DecColumn
-    # declErr: not available yet DM-15180
-    # ra_decl_Cov: not available yet
+    # decErr: not available yet DM-15180
+    # ra_dec_Cov: not available yet
     x:
         functor: Column
         args: slot_Centroid_x
@@ -65,27 +69,27 @@ funcs:
     snr:
         functor: Column
         args: snr
-    psFlux:
+    psfFlux:
         functor: LocalNanojansky
         args:
             - slot_PsfFlux_instFlux
             - slot_PsfFlux_instFluxErr
             - base_LocalPhotoCalib
             - base_LocalPhotoCalibErr
-    psFluxErr:
+    psfFluxErr:
         functor: LocalNanojanskyErr
         args:
             - slot_PsfFlux_instFlux
             - slot_PsfFlux_instFluxErr
             - base_LocalPhotoCalib
             - base_LocalPhotoCalibErr
-    # ps_ra_cov not implemented
-    # ps_dec_cov not implemented
-    # psLnl not implemented
-    psChi2:
+    # psfFlux_psfRa_cov not implemented
+    # psfFlux_psfDec_cov not implemented
+    # psfLnl not implemented
+    psfChi2:
         functor: Column
         args: slot_PsfFlux_chi2
-    psNdata:
+    psfNdata:
         functor: Column
         args: slot_PsfFlux_npixels
     trailFlux:
@@ -98,7 +102,7 @@ funcs:
     trailRa:
         functor: Column
         args: ext_trailedSources_Naive_ra
-    trailDecl:
+    trailDec:
         functor: Column
         args: ext_trailedSources_Naive_dec
     trailLength:
@@ -116,7 +120,7 @@ funcs:
     # trailLnL not implemented
     # trailChi2 not implemented
     # trailNdata not implemented
-    dipMeanFlux:
+    dipoleMeanFlux:
         functor: LocalDipoleMeanFlux
         args:
             - ip_diffim_NaiveDipoleFlux_pos_instFlux
@@ -125,7 +129,7 @@ funcs:
             - ip_diffim_NaiveDipoleFlux_neg_instFluxErr
             - base_LocalPhotoCalib
             - base_LocalPhotoCalibErr
-    dipMeanFluxErr:
+    dipoleMeanFluxErr:
         functor: LocalDipoleMeanFluxErr
         args:
             - ip_diffim_NaiveDipoleFlux_pos_instFlux
@@ -134,7 +138,7 @@ funcs:
             - ip_diffim_NaiveDipoleFlux_neg_instFluxErr
             - base_LocalPhotoCalib
             - base_LocalPhotoCalibErr
-    dipFluxDiff:
+    dipoleFluxDiff:
         functor: LocalDipoleDiffFlux
         args:
             - ip_diffim_NaiveDipoleFlux_pos_instFlux
@@ -143,7 +147,7 @@ funcs:
             - ip_diffim_NaiveDipoleFlux_neg_instFluxErr
             - base_LocalPhotoCalib
             - base_LocalPhotoCalibErr
-    dipFluxDiffErr:
+    dipoleFluxDiffErr:
         functor: LocalDipoleDiffFluxErr
         args:
             - ip_diffim_NaiveDipoleFlux_pos_instFlux
@@ -152,46 +156,46 @@ funcs:
             - ip_diffim_NaiveDipoleFlux_neg_instFluxErr
             - base_LocalPhotoCalib
             - base_LocalPhotoCalibErr
-    # dipRa not implemented
-    # dipDec not implemented
+    # dipoleRa not implemented
+    # dipoleDec not implemented
     #    (this may be redundant with RA/DEC as the default centroid is the
     #     dip model, defaulting to SdssCentroid on Dip-Fit failure.)
-    dipLength:
+    dipoleLength:
         functor: ConvertPixelToArcseconds
         args:
             - ip_diffim_DipoleFit_separation
             - base_LocalWcs_CDMatrix_1_1
             - base_LocalWcs_CDMatrix_1_2
             - base_LocalWcs_CDMatrix_2_1
             - base_LocalWcs_CDMatrix_2_2
-    dipAngle:
+    dipoleAngle:
         functor: Column
         args: ip_diffim_DipoleFit_orientation
-    # dipCov not implemented
-    # dipLnl not implemented
-    dipChi2:
+    # dipoleCov not implemented
+    # dipoleLnl not implemented
+    dipoleChi2:
         functor: Column
         args: ip_diffim_DipoleFit_chi2dof
     isDipole:
         functor: Column
         args: ip_diffim_DipoleFit_flag_classification
-    # dipNdata not implemented
-    totFlux:
+    # dipoleNdata not implemented
+    scienceFlux:
         functor: LocalNanojansky
         args:
             - ip_diffim_forced_PsfFlux_instFlux
             - ip_diffim_forced_PsfFlux_instFluxErr
             - base_LocalPhotoCalib
             - base_LocalPhotoCalibErr
-    totFluxErr:
+    scienceFluxErr:
         functor: LocalNanojanskyErr
         args:
             - ip_diffim_forced_PsfFlux_instFlux
             - ip_diffim_forced_PsfFlux_instFluxErr
             - base_LocalPhotoCalib
             - base_LocalPhotoCalibErr
-    # diffFlux not implemented and likely dropped due to no snaps.
-    # diffFluxErr not implemented and likely dropped due to no snaps.
+    # snapDiffFlux not implemented and likely dropped due to no snaps.
+    # snapDiffFluxErr not implemented and likely dropped due to no snaps.
     # fpBkgd not measured yet. DM-
     # fpBkgdErr not measured yet. DM-
 
@@ -247,6 +251,6 @@ funcs:
             - base_LocalWcs_CDMatrix_2_1
             - base_LocalWcs_CDMatrix_2_2
     # extendedness not implemented
-    spuriousness:
+    reliability:
         functor: Column
-        args: spuriousness
+        args: reliability
diff --git a/data/apdb-ap-pipe-afw-map.yaml b/data/apdb-ap-pipe-afw-map.yaml
@@ -6,10 +6,10 @@
 DiaObject:
   diaObjectId: id
   ra: coord_ra
-  decl: coord_dec
+  dec: coord_dec
 
 DiaSource:
   diaSourceId: id
   parentDiaSourceId: parent
   ra: coord_ra
-  decl: coord_dec
+  dec: coord_dec
diff --git a/python/lsst/ap/association/association.py b/python/lsst/ap/association/association.py
@@ -127,7 +127,7 @@ def check_dia_source_radec(self, dia_sources):
             RA/DEC.
         """
         nan_mask = (dia_sources.loc[:, "ra"].isnull()
-                    | dia_sources.loc[:, "decl"].isnull())
+                    | dia_sources.loc[:, "dec"].isnull())
         if np.any(nan_mask):
             nan_idxs = np.argwhere(nan_mask.to_numpy()).flatten()
             for nan_idx in nan_idxs:
@@ -264,7 +264,7 @@ def _radec_to_xyz(self, catalog):
             Output unit-vectors
         """
         ras = np.radians(catalog["ra"])
-        decs = np.radians(catalog["decl"])
+        decs = np.radians(catalog["dec"])
         vectors = np.empty((len(ras), 3))
 
         sin_dec = np.sin(np.pi / 2 - decs)

diff --git a/python/lsst/ap/association/diaForcedSource.py b/python/lsst/ap/association/diaForcedSource.py
@@ -198,7 +198,7 @@ def _convert_from_pandas(self, input_objects):
             outputRecord.setId(obj_id)
             outputRecord.setCoord(
                 geom.SpherePoint(df_row["ra"],
-                                 df_row["decl"],
+                                 df_row["dec"],
                                  geom.degrees))
         return outputCatalog
 
@@ -236,23 +236,24 @@ def _calibrate_and_merge(self,
 
         output_catalog = diff_sources.asAstropy().to_pandas()
         output_catalog.rename(columns={"id": "diaForcedSourceId",
-                                       "slot_PsfFlux_instFlux": "psFlux",
-                                       "slot_PsfFlux_instFluxErr": "psFluxErr",
+                                       "slot_PsfFlux_instFlux": "psfFlux",
+                                       "slot_PsfFlux_instFluxErr": "psfFluxErr",
                                        "slot_Centroid_x": "x",
                                        "slot_Centroid_y": "y"},
                               inplace=True)
-        output_catalog.loc[:, "psFlux"] = diff_fluxes[:, 0]
-        output_catalog.loc[:, "psFluxErr"] = diff_fluxes[:, 1]
+        output_catalog.loc[:, "psfFlux"] = diff_fluxes[:, 0]
+        output_catalog.loc[:, "psfFluxErr"] = diff_fluxes[:, 1]
 
-        output_catalog["totFlux"] = direct_fluxes[:, 0]
-        output_catalog["totFluxErr"] = direct_fluxes[:, 1]
+        output_catalog["scienceFlux"] = direct_fluxes[:, 0]
+        output_catalog["scienceFluxErr"] = direct_fluxes[:, 1]
 
-        visit_info = direct_exp.getInfo().getVisitInfo()
+        visit_info = direct_exp.visitInfo
         ccdVisitId = direct_exp.info.id
-        midPointTaiMJD = visit_info.getDate().get(system=DateTime.MJD)
+        midpointMjdTai = visit_info.date.get(system=DateTime.MJD)
         output_catalog["ccdVisitId"] = ccdVisitId
-        output_catalog["midPointTai"] = midPointTaiMJD
-        output_catalog["filterName"] = diff_exp.getFilter().bandLabel
+        output_catalog["midpointMjdTai"] = midpointMjdTai
+        output_catalog["band"] = diff_exp.getFilter().bandLabel
+        output_catalog["time_processed"] = DateTime.now().toPython()
 
         # Drop superfluous columns from output DataFrame.
         output_catalog.drop(columns=self.config.dropColumns, inplace=True)

diff --git a/python/lsst/ap/association/diaPipe.py b/python/lsst/ap/association/diaPipe.py
@@ -402,7 +402,7 @@ def run(self,
         updatedDiaObjectIds = associatedDiaSources["diaObjectId"][
             associatedDiaSources["diaObjectId"] != 0].to_numpy()
         associatedDiaSources.set_index(["diaObjectId",
-                                        "filterName",
+                                        "band",
                                         "diaSourceId"],
                                        drop=False,
                                        inplace=True)
@@ -468,7 +468,7 @@ def run(self,
         # Store DiaSources, updated DiaObjects, and DiaForcedSources in the
         # Apdb.
         self.apdb.store(
-            exposure.getInfo().getVisitInfo().getDate(),
+            exposure.visitInfo.date,
             diaCalResult.updatedDiaObjects,
             associatedDiaSources,
             diaForcedSources)
@@ -562,7 +562,7 @@ def _initialize_dia_object(self, objId):
                 Id of the a nearbyObject in the Object table (`int`).
             ``nearbyObj3``
                 Id of the a nearbyObject in the Object table (`int`).
-            ``?PSFluxData``
+            ``?_psfFluxNdata``
                 Number of data points used to calculate point source flux
                 summary statistics in each bandpass (`int`).
         """
@@ -573,7 +573,7 @@ def _initialize_dia_object(self, objId):
                           "nearbyObj3": 0,
                           "flags": 0}
         for f in ["u", "g", "r", "i", "z", "y"]:
-            new_dia_object["%sPSFluxNdata" % f] = 0
+            new_dia_object["%s_psfFluxNdata" % f] = 0
         return pd.Series(data=new_dia_object)
 
     def testDataFrameIndex(self, df):

diff --git a/python/lsst/ap/association/loadDiaCatalogs.py b/python/lsst/ap/association/loadDiaCatalogs.py
@@ -77,13 +77,12 @@ def run(self, exposure, apdb):
               the ``diaObjectId`` column. (`pandas.DataFrame`)
             - ``diaSources`` : Complete set of DiaSources covering the input
               exposure padded by ``pixelMargin``. DataFrame is indexed by
-              ``diaObjectId``, ``filterName``, ``diaSourceId`` columns.
+              ``diaObjectId``, ``band``, ``diaSourceId`` columns.
               (`pandas.DataFrame`)
         """
-        visiInfo = exposure.getInfo().getVisitInfo()
         region = self._getRegion(exposure)
 
-        # This is the first database query
+        # This is the first database query.
         try:
             diaObjects = self.loadDiaObjects(region, apdb)
         except (OperationalError, ProgrammingError) as e:
@@ -92,17 +91,11 @@ def run(self, exposure, apdb):
                 "make_apdb.py first? If you did, some other error occurred "
                 "during database access of the DiaObject table.") from e
 
-        dateTime = visiInfo.getDate()
+        dateTime = exposure.visitInfo.date
 
-        diaSources = self.loadDiaSources(diaObjects,
-                                         region,
-                                         dateTime,
-                                         apdb)
+        diaSources = self.loadDiaSources(diaObjects, region, dateTime, apdb)
 
-        diaForcedSources = self.loadDiaForcedSources(diaObjects,
-                                                     region,
-                                                     dateTime,
-                                                     apdb)
+        diaForcedSources = self.loadDiaForcedSources(diaObjects, region, dateTime, apdb)
 
         return pipeBase.Struct(
             diaObjects=diaObjects,
@@ -172,12 +165,12 @@ def loadDiaSources(self, diaObjects, region, dateTime, apdb):
             # If no area is specified return an empty DataFrame with the
             # the column used for indexing later in AssociationTask.
             diaSources = pd.DataFrame(columns=["diaObjectId",
-                                               "filterName",
+                                               "band",
                                                "diaSourceId"])
         else:
             diaSources = apdb.getDiaSources(region, diaObjects.loc[:, "diaObjectId"], dateTime)
 
-        diaSources.set_index(["diaObjectId", "filterName", "diaSourceId"],
+        diaSources.set_index(["diaObjectId", "band", "diaSourceId"],
                              drop=False,
                              inplace=True)
         if diaSources.index.has_duplicates:
@@ -187,7 +180,7 @@ def loadDiaSources(self, diaObjects, region, dateTime, apdb):
             # Drop duplicates via index and keep the first appearance. Reset
             # due to the index shape being slight different thatn expected.
             diaSources = diaSources.groupby(diaSources.index).first().reset_index(drop=True)
-            diaSources.set_index(["diaObjectId", "filterName", "diaSourceId"],
+            diaSources.set_index(["diaObjectId", "band", "diaSourceId"],
                                  drop=False,
                                  inplace=True)
 
@@ -233,7 +226,7 @@ def loadDiaForcedSources(self, diaObjects, region, dateTime, apdb):
                 "Duplicate DiaForcedSources loaded from the Apdb. This may "
                 "cause downstream pipeline issues. Dropping duplicated rows.")
             # Drop duplicates via index and keep the first appearance. Reset
-            # due to the index shape being slight different thatn expected.
+            # due to the index shape being slightly different than expected.
             diaForcedSources = diaForcedSources.groupby(diaForcedSources.index).first()
             diaForcedSources.reset_index(drop=True, inplace=True)
             diaForcedSources.set_index(["diaObjectId", "diaForcedSourceId"],

diff --git a/python/lsst/ap/association/packageAlerts.py b/python/lsst/ap/association/packageAlerts.py
@@ -96,14 +96,14 @@ def run(self,
         ----------
         diaSourceCat : `pandas.DataFrame`
             New DiaSources to package. DataFrame should be indexed on
-            ``["diaObjectId", "filterName", "diaSourceId"]``
+            ``["diaObjectId", "band", "diaSourceId"]``
         diaObjectCat : `pandas.DataFrame`
             New and updated DiaObjects matched to the new DiaSources. DataFrame
             is indexed on ``["diaObjectId"]``
         diaSrcHistory : `pandas.DataFrame`
             12 month history of DiaSources matched to the DiaObjects. Excludes
             the newest DiaSource and is indexed on
-            ``["diaObjectId", "filterName", "diaSourceId"]``
+            ``["diaObjectId", "band", "diaSourceId"]``
         diaForcedSources : `pandas.DataFrame`
             12 month history of DiaForcedSources matched to the DiaObjects.
             ``["diaObjectId"]``
@@ -138,7 +138,7 @@ def run(self,
                 objSourceHistory = None
             objDiaForcedSources = diaForcedSources.loc[srcIndex[0]]
             sphPoint = geom.SpherePoint(diaSource["ra"],
-                                        diaSource["decl"],
+                                        diaSource["dec"],
                                         geom.degrees)
 
             cutoutExtent = self.createDiaSourceExtent(diaSource["bboxSize"])