From b47955f8abc8d9d0d59941ddf2f97b226829701f Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Mon, 11 Sep 2023 15:10:18 -0700 Subject: [PATCH 1/3] Add fgcmcal to LATISS pipeline, and split steps accordingly. --- pipelines/LATISS/DRP.yaml | 78 +++++++++++++++++++++++++++++++++++---- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml index 3fc424e1..762ea38e 100644 --- a/pipelines/LATISS/DRP.yaml +++ b/pipelines/LATISS/DRP.yaml @@ -1,20 +1,25 @@ description: DRP specialized for LATISS surveys instrument: lsst.obs.lsst.Latiss imports: - - location: $DRP_PIPE_DIR/pipelines/_ingredients/DRP-minimal-calibration.yaml + - location: $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml include: - isr - characterizeImage - calibrate - - writeSourceTable - - transformSourceTable - - consolidateSourceTable + - writePreSourceTable + - transformPreSourceTable + - consolidatePreSourceTable - consolidateVisitSummary - isolatedStarAssociation + - fgcmBuildFromIsolatedStars + - fgcmFitCycle + - fgcmOutputProducts - finalizeCharacterization - updateVisitSummary - makeCcdVisitTable - makeVisitTable + - transformSourceTable + - writeRecalibratedSourceTable - makeWarp - assembleCoadd - detection @@ -27,14 +32,26 @@ imports: - consolidateObjectTable - forcedPhotCoadd +tasks: + fgcmFitCycle: + class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask + config: + doMultipleCycles: true + multipleCyclesFinalCycleNumber: 6 + + fgcmOutputProducts: + class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask + config: + connections.cycleNumber: 6 + subsets: step1: subset: - isr - characterizeImage - calibrate - - writeSourceTable - - transformSourceTable + - writePreSourceTable + - transformPreSourceTable description: | Per-detector tasks that can be run together to start the DRP pipeline. These should never be run with 'tract' or 'patch' as part of the data ID @@ -44,15 +61,47 @@ subsets: this may be moot for a single-detector camera, we would still like to follow best practices envisioned for the full LSST survey). - step2: + step2a: subset: - - consolidateSourceTable + - consolidatePreSourceTable - consolidateVisitSummary - isolatedStarAssociation + description: | + Global tasks: This is a mix of visit-level and tract-level tasks that + must be run with a skymap data query constraint only (an instrument + constraint is fine, but generally unnecessary). For example, running + with 'tract' (and 'patch') constraints will select partial visits that + overlap that region. A skymap constraint is necessary to prevent + isolatedStarAssociation from producing results for all skymaps in the + data repository. + + Visit-level: consolidatePreSourceTable, consolidateVisitSummary + Tract-level: isolatedStarAssociation + + step2bcde: + subset: - finalizeCharacterization + - fgcmBuildFromIsolatedStars + - fgcmFitCycle + - fgcmOutputProducts - updateVisitSummary - makeCcdVisitTable - makeVisitTable + description: | + Per-visit and per-collection tasks that can be run together after step2a + with no data query constraints other than instrument. + + FGCM requires full visits and 'tract' and 'patch' constraints will + always select partial visits that overlap that region. + + This includes FGCM because it's configured here to run in "global" mode, + which means one should not use a 'tract' expression to constrain it, and + if one _did_ run it with a tract constraint (which would be a common + occurrence if it was included in any later step). + + step2: + subset: + - consolidateSourceTable description: | This is a mix of visit-level and tract-level tasks that must not be run with data query constraints other than instrument. These are tasks that @@ -93,3 +142,16 @@ subsets: 'step3a' subsets (seperation into step3a and step3b is necessary for now as otherwise failues in upstream tasks will cause these to fail. This is hoped to be a temporary limitation). + + step4: + subset: + - writeRecalibratedSourceTable + - transformSourceTable + - consolidateSourceTable + description: | + Tasks that be run together, but only after the 'step1', 'step2', and + 'step3' subsets. + + These detector-level tasks should not be run with 'tract' or 'patch' as + part of the data ID expression if all reference catalogs are desired, + and to ensure that consolidateSourceTable contains complete visits. From a8d5d22e5fa0ffce1f470ca1611d0c3d3cc561a0 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 12 Sep 2023 09:51:01 -0700 Subject: [PATCH 2/3] Remove old step2 and make sure consolidateSourceTable is imported. --- pipelines/LATISS/DRP.yaml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml index 762ea38e..ba843583 100644 --- a/pipelines/LATISS/DRP.yaml +++ b/pipelines/LATISS/DRP.yaml @@ -19,6 +19,7 @@ imports: - makeCcdVisitTable - makeVisitTable - transformSourceTable + - consolidateSourceTable - writeRecalibratedSourceTable - makeWarp - assembleCoadd @@ -99,19 +100,6 @@ subsets: if one _did_ run it with a tract constraint (which would be a common occurrence if it was included in any later step). - step2: - subset: - - consolidateSourceTable - description: | - This is a mix of visit-level and tract-level tasks that must not be run - with data query constraints other than instrument. These are tasks that - can be run together, but only after the 'step1' subset. - - Visit-level: consolidateSourceTable, consolidateVisitSummary, - finalizeCharacterization, updateVisitSummary, - makeCcdVisitTable, makVisitTable - Tract-level: isolatedStarAssociation - step3a: subset: - makeWarp From a1ccc5387afeb176293d3fb540c540c69d43a444 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Tue, 12 Sep 2023 15:42:49 -0700 Subject: [PATCH 3/3] Override updateVisitSummary config to only import fgcmcal calibrations. --- pipelines/LATISS/DRP.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml index ba843583..71c473ad 100644 --- a/pipelines/LATISS/DRP.yaml +++ b/pipelines/LATISS/DRP.yaml @@ -45,6 +45,13 @@ tasks: config: connections.cycleNumber: 6 + updateVisitSummary: + class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask + config: + wcs_provider: "input_summary" + photo_calib_provider: "global" + background_provider: "input_summary" + subsets: step1: subset: