diff --git a/data/flag-rename-rules.yaml b/data/flag-rename-rules.yaml index 26d99a88..2ed5a58a 100644 --- a/data/flag-rename-rules.yaml +++ b/data/flag-rename-rules.yaml @@ -5,3 +5,4 @@ flag_rename_rules: - ['slot_Centroid', 'centroid'] - ['slot_PsfFlux', 'psfFlux'] - ['slot_Shape', 'shape'] + - ['ext_trailedSources_Naive', 'trail'] diff --git a/python/lsst/ap/association/diaForcedSource.py b/python/lsst/ap/association/diaForcedSource.py index 0163b42c..5d4955c6 100644 --- a/python/lsst/ap/association/diaForcedSource.py +++ b/python/lsst/ap/association/diaForcedSource.py @@ -235,7 +235,6 @@ def _calibrate_and_merge(self, output_catalog["band"] = diff_exp.getFilter().bandLabel output_catalog["time_processed"] = DateTime.now().toPython() # TODO: propagate actual flags (DM-42355) - output_catalog["flags"] = 0 # Drop superfluous columns from output DataFrame. output_catalog.drop(columns=self.config.dropColumns, inplace=True) diff --git a/python/lsst/ap/association/diaPipe.py b/python/lsst/ap/association/diaPipe.py index f22d5bff..9ee2d556 100644 --- a/python/lsst/ap/association/diaPipe.py +++ b/python/lsst/ap/association/diaPipe.py @@ -321,7 +321,6 @@ def setDefaults(self): self.apdb.dia_object_columns = [] self.diaCalculation.plugins = ["ap_meanPosition", "ap_nDiaSources", - "ap_diaObjectFlag", "ap_meanFlux", "ap_percentileFlux", "ap_sigmaFlux", @@ -564,7 +563,7 @@ def run(self, # alertPackager needs correct columns diaForcedSources = pd.DataFrame(columns=[ "diaForcedSourceId", "diaObjectID", "ccdVisitID", "psfFlux", "psfFluxErr", - "x", "y", "flags", "midpointMjdTai", "band", + "x", "y", "midpointMjdTai", "band", ]) # Store DiaSources, updated DiaObjects, and DiaForcedSources in the @@ -695,8 +694,7 @@ def _initialize_dia_object(self, objId): "pmParallaxNdata": 0, "nearbyObj1": 0, "nearbyObj2": 0, - "nearbyObj3": 0, - "flags": 0} + "nearbyObj3": 0} for f in ["u", "g", "r", "i", "z", "y"]: new_dia_object["%s_psfFluxNdata" % f] = 0 return pd.Series(data=new_dia_object) diff --git a/python/lsst/ap/association/trailedSourceFilter.py b/python/lsst/ap/association/trailedSourceFilter.py index d7721928..8408ffb2 100644 --- a/python/lsst/ap/association/trailedSourceFilter.py +++ b/python/lsst/ap/association/trailedSourceFilter.py @@ -85,17 +85,21 @@ def run(self, dia_sources, exposure_time): (`pandas.DataFrame`) """ - flag_map = os.path.join(utils.getPackageDir("ap_association"), "data/association-flag-map.yaml") - unpacker = UnpackApdbFlags(flag_map, "DiaSource") - flags = unpacker.unpack(dia_sources["flags"], "flags") + if "flags" in dia_sources.columns: + flag_map = os.path.join(utils.getPackageDir("ap_association"), "data/association-flag-map.yaml") + unpacker = UnpackApdbFlags(flag_map, "DiaSource") + flags = unpacker.unpack(dia_sources["flags"], "flags") + trail_edge_flags = flags["ext_trailedSources_Naive_flag_edge"] + else: + trail_edge_flags = dia_sources["trail_flag_edge"] - trail_mask = self._check_dia_source_trail(dia_sources, exposure_time, flags) + trail_mask = self._check_dia_source_trail(dia_sources, exposure_time, trail_edge_flags) return pipeBase.Struct( diaSources=dia_sources[~trail_mask].reset_index(drop=True), longTrailedDiaSources=dia_sources[trail_mask].reset_index(drop=True)) - def _check_dia_source_trail(self, dia_sources, exposure_time, flags): + def _check_dia_source_trail(self, dia_sources, exposure_time, trail_edge_flags): """Find DiaSources that have long trails. Return a mask of sources with lengths greater than @@ -108,18 +112,18 @@ def _check_dia_source_trail(self, dia_sources, exposure_time, flags): Input DIASources to check for trail lengths. exposure_time : `float` Exposure time from difference image. - flags : 'numpy.ndArray' - Boolean array of flags from the DIASources. + trail_edge_flags : 'numpy.ndArray' + Boolean array of trail_flag_edge flags from the DIASources. Returns ------- trail_mask : `pandas.DataFrame` Boolean mask for DIASources which are greater than the - cutoff length and have the edge flag set. + cutoff length or have the edge flag set. """ trail_mask = (dia_sources.loc[:, "trailLength"].values[:] >= (self.config.max_trail_length*exposure_time)) - trail_mask[np.where(flags['ext_trailedSources_Naive_flag_edge'])] = True + trail_mask[np.where(trail_edge_flags)] = True return trail_mask diff --git a/python/lsst/ap/association/transformDiaSourceCatalog.py b/python/lsst/ap/association/transformDiaSourceCatalog.py index 333517a0..b3d60c7b 100644 --- a/python/lsst/ap/association/transformDiaSourceCatalog.py +++ b/python/lsst/ap/association/transformDiaSourceCatalog.py @@ -99,11 +99,13 @@ class TransformDiaSourceCatalogConfig(TransformCatalogBaseConfig, doc="Input DiaSource catalog contains SkySources that should be " "removed before storing the output DiaSource catalog." ) + # TODO: remove on DM-41532 doPackFlags = pexConfig.Field( dtype=bool, - default=True, + default=False, doc="Do pack the flags into one integer column named 'flags'." - "If False, instead produce one boolean column per flag." + "If False, instead produce one boolean column per flag.", + deprecated="This field is no longer used. Will be removed after v28." ) doIncludeReliability = pexConfig.Field( dtype=bool, diff --git a/tests/test_diaForcedSource.py b/tests/test_diaForcedSource.py index c8277be1..aedc5cef 100644 --- a/tests/test_diaForcedSource.py +++ b/tests/test_diaForcedSource.py @@ -173,7 +173,7 @@ def setUp(self): # above list of ids. self.expectedDiaForcedSources = 6 - self.expected_n_columns = 14 + self.expected_n_columns = 13 def testRun(self): """Test that forced source catalogs are successfully created and have diff --git a/tests/test_loadDiaCatalogs.py b/tests/test_loadDiaCatalogs.py index 372da857..64f319e7 100644 --- a/tests/test_loadDiaCatalogs.py +++ b/tests/test_loadDiaCatalogs.py @@ -81,7 +81,7 @@ def setUp(self): # These columns are not in the DPDD, yet do appear in DiaSource.yaml. # We don't need to check them against the default APDB schema. - self.ignoreColumns = ["band", "bboxSize", "isDipole"] + self.ignoreColumns = ["band", "bboxSize", "isDipole", "flags"] def testRun(self): """Test the full run method for the loader. diff --git a/tests/test_packageAlerts.py b/tests/test_packageAlerts.py index 3765c569..bb041250 100644 --- a/tests/test_packageAlerts.py +++ b/tests/test_packageAlerts.py @@ -126,7 +126,6 @@ def mock_alert(alert_id): "snr": np.float32(6.7), "psfFlux": np.float32(700.0), "psfFluxErr": np.float32(90.0), - "flags": 12345, } } @@ -197,10 +196,6 @@ def setUp(self): diaSourceHistory, diaForcedSources, self.exposure.visitInfo.date.toAstropy()) - self.diaObjects.replace(to_replace=[None], value=np.nan, inplace=True) - diaSourceHistory.replace(to_replace=[None], value=np.nan, inplace=True) - self.diaForcedSources.replace(to_replace=[None], value=np.nan, - inplace=True) diaSourceHistory["programId"] = 0 self.diaSources = diaSourceHistory.loc[[(1, "g", 9), (2, "g", 10)], :] diff --git a/tests/test_transformDiaSourceCatalog.py b/tests/test_transformDiaSourceCatalog.py index 3ae66f94..812ed4ec 100644 --- a/tests/test_transformDiaSourceCatalog.py +++ b/tests/test_transformDiaSourceCatalog.py @@ -177,9 +177,11 @@ def test_computeBBoxSize(self): self.assertEqual(size, self.bboxSize) self.assertEqual(len(boxSizes), self.nSources) + # TODO: remove in DM-41532 def test_bit_unpacker(self): """Test that the integer bit packer is functioning correctly. """ + self.config.doPackFlags = True transform = TransformDiaSourceCatalogTask(initInputs=self.initInputs, config=self.config) for idx, obj in enumerate(self.inputCatalog): diff --git a/tests/utils_tests.py b/tests/utils_tests.py index c41ad84e..9d6c693e 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -65,7 +65,6 @@ def makeDiaObjects(nObjects, exposure, rng): "nearbyObj1": 0, "nearbyObj2": 0, "nearbyObj3": 0, - "flags": 1, "nDiaSources": 5} for f in ["u", "g", "r", "i", "z", "y"]: newObject["%s_psfFluxNdata" % f] = 0 @@ -124,8 +123,7 @@ def makeDiaSources(nSources, diaObjectIds, exposure, rng, randomizeObjects=False "band": exposure.getFilter().bandLabel, "psfNdata": 0, "trailNdata": 0, - "dipoleNdata": 0, - "flags": 1}) + "dipoleNdata": 0}) return pd.DataFrame(data=data) @@ -167,8 +165,7 @@ def makeDiaForcedSources(nForcedSources, diaObjectIds, exposure, rng, randomizeO "diaObjectId": objId, "midpointMjdTai": midpointMjdTai + 1.0 * i, "time_processed": datetime.datetime.now(), - "band": exposure.getFilter().bandLabel, - "flags": 0}) + "band": exposure.getFilter().bandLabel}) return pd.DataFrame(data=data)