ApPipe.yaml

description: End to end Alert Production pipeline.
# Look in subdirectories of $AP_PIPE_DIR/pipelines to find customized pipelines
# for each camera. Those pipelines import this general AP pipeline.
#
# Per RFC-997, all variants of ApPipe must define three mutually exclusive subsets:
# - preload is tasks that can be run before raw images arrive
# - prompt is everything done by Prompt Processing starting from raws
# - afterburner is metrics and other non-essential tasks that are skipped by Prompt Processing
#
# NOTES
# Remember to run `apdb-cli create-sql`
# Option to specify connection_timeout for sqlite APDBs encountering lock errors, i.e.,
# --connection_timeout 240

imports:
  - $ANALYSIS_TOOLS_DIR/pipelines/apDetectorVisitQualityCore.yaml
parameters:
  # Pipeline configurable to run on both goodSeeing and deep templates, depending on dataset.
  coaddName: goodSeeing
  # APDB config file must be user-specified
  apdb_config:
tasks:
  loadDiaCatalogs:
    class: lsst.ap.association.LoadDiaCatalogsTask
    config:
      apdb_config_url: parameters.apdb_config
  isr:
    class: lsst.ip.isr.IsrTask
    config:
      doBrighterFatter: False
  calibrateImage:
    class: lsst.pipe.tasks.calibrateImage.CalibrateImageTask
  retrieveTemplate:
    class: lsst.ip.diffim.getTemplate.GetTemplateTask
    config:
      connections.bbox: initial_pvi.bbox
      connections.wcs: initial_pvi.wcs
      connections.coaddName: parameters.coaddName
  subtractImages:
    class: lsst.ip.diffim.subtractImages.AlardLuptonSubtractTask
    config:
      connections.science: initial_pvi
      connections.sources: initial_stars_footprints_detector
      connections.coaddName: parameters.coaddName
      doApplyExternalCalibrations: False
  detectAndMeasure:
    class: lsst.ip.diffim.detectAndMeasure.DetectAndMeasureTask
    config:
      connections.science: initial_pvi
      connections.coaddName: parameters.coaddName
      doSkySources: True
  filterDiaSrcCat:
    class: lsst.ap.association.FilterDiaSourceCatalogTask
    config:
      doRemoveSkySources: True
      connections.coaddName: parameters.coaddName
  rbClassify:
    class: lsst.meas.transiNet.RBTransiNetTask
    config:
      modelPackageStorageMode: butler
      connections.science: initial_pvi
      connections.coaddName: parameters.coaddName
  transformDiaSrcCat:
    class: lsst.ap.association.TransformDiaSourceCatalogTask
    config:
      doRemoveSkySources: True
      connections.coaddName: parameters.coaddName
      doIncludeReliability: True  # Output from rbClassify
      doUseApdbSchema: True  # Force the table columns to match the APDB definition in sdm_schemas
  getRegionTimeFromVisit:
    class: lsst.pipe.tasks.getRegionTimeFromVisit.GetRegionTimeFromVisitTask
    config:
      connections.coaddName: parameters.coaddName
  mpSkyEphemerisQuery:
    class: lsst.ap.association.MPSkyEphemerisQueryTask
  diaPipe:
    class: lsst.ap.association.DiaPipelineTask
    config:
      apdb_config_url: parameters.apdb_config
      doPackageAlerts: True  # Test alert generation, but don't output
      alertPackager.useAveragePsf: True  # Speed up production processing; don't want as default or in ApPipeWithFakes
      connections.exposure: initial_pvi
      connections.coaddName: parameters.coaddName
  sampleSpatialMetrics:
    class: lsst.ip.diffim.SpatiallySampledMetricsTask
    config:
      connections.science: initial_pvi
      connections.coaddName: parameters.coaddName

subsets:
  apPipeSingleFrame:
    subset:
      - isr
      - calibrateImage
      - initialPviCore
    description: >-
      The prompt ApPipe tasks that make up single-frame processing. Not to be confused with the
      SingleFrame.yaml pipeline, which does more than just ApPipe single frame processing, and
      is designed as a standalone alternative to ApPipe.yaml.
  processCcd:  # TODO: remove on DM-48428
    subset:
      - isr
      - calibrateImage
      - initialPviCore
    description: Deprecated alias for apPipeSingleFrame, will be removed after v29.
  apPipe:
    subset:
      - loadDiaCatalogs
      - isr
      - calibrateImage
      - retrieveTemplate
      - subtractImages
      - detectAndMeasure
      - filterDiaSrcCat
      - rbClassify
      - transformDiaSrcCat
      - getRegionTimeFromVisit
      - mpSkyEphemerisQuery
      - diaPipe
      - analyzeAssocDiaSrcCore
      - analyzeTrailedDiaSrcCore
      - sampleSpatialMetrics
      - diffimTaskCore
      - detectionTaskCore
      - associationCore
      - associationTiming
      - diffimTaskPlots
      - initialPviCore
    description: >
      An alias of ApPipe to use in higher-level pipelines.
  preload:
    subset:
      - loadDiaCatalogs
      - mpSkyEphemerisQuery
    description: Tasks that can be run before receiving raw images.
  prompt:
    subset:
      - isr
      - calibrateImage
      - retrieveTemplate
      - subtractImages
      - detectAndMeasure
      - filterDiaSrcCat
      - rbClassify
      - transformDiaSrcCat
      - diaPipe
      - analyzeAssocDiaSrcCore
      - analyzeTrailedDiaSrcCore
      - diffimTaskCore
      - detectionTaskCore
      - associationCore
      - associationTiming
      - initialPviCore
    description: >
      Tasks necessary to turn raw images into APDB rows and alerts.
      Requires preload subset to be run first.
  afterburner:
    subset:
      - sampleSpatialMetrics
      - diffimTaskPlots
    description: >
      Tasks for QA and other non-real-time processing.
      Requires prompt subset to be run first.
contracts:
  - detectAndMeasure.doSkySources == transformDiaSrcCat.doRemoveSkySources
  # Both loadDiaCatalogs and diaPipe connect to the APDB, so make sure they use the same configuration
  - loadDiaCatalogs.apdb_config_url == diaPipe.apdb_config_url
  # Inputs and outputs must match. For consistency, contracts are written in execution order:
  #     first task == second task, then sorted by (first, second)
  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
  - contract: loadDiaCatalogs.connections.ConnectionsClass(config=loadDiaCatalogs).diaObjects.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).preloadedDiaObjects.name
    msg: "loadDiaCatalogs.diaObjects != diaPipe.preloadedDiaObjects"
  - contract: loadDiaCatalogs.connections.ConnectionsClass(config=loadDiaCatalogs).diaSources.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).preloadedDiaSources.name
    msg: "loadDiaCatalogs.diaSources != diaPipe.preloadedDiaSources"
  - contract: loadDiaCatalogs.connections.ConnectionsClass(config=loadDiaCatalogs).diaForcedSources.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).preloadedDiaForcedSources.name
    msg: "loadDiaCatalogs.diaForcedSources != diaPipe.preloadedDiaForcedSources"
  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).exposure.name ==
              subtractImages.connections.ConnectionsClass(config=subtractImages).science.name
    msg: "calibrateImage.exposure != subtractImages.science"
  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).stars_footprints.name ==
              subtractImages.connections.ConnectionsClass(config=subtractImages).sources.name
    msg: "calibrateImage.footprints_stars != subtractImages.sources"
  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).exposure.name ==
              rbClassify.connections.ConnectionsClass(config=rbClassify).science.name
    msg: "calibrateImage.exposure != rbClassify.science"
  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).exposure.name ==
              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).science.name
    msg: "calibrateImage.exposure != sampleSpatialMetrics.science"
  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).exposure.name + ".summaryStats" ==
              initialPviCore.connections.ConnectionsClass(config=initialPviCore).data.name
    msg: "calibrateImage.exposure != initialPviCore.data"
  - contract: retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
              subtractImages.connections.ConnectionsClass(config=subtractImages).template.name
    msg: "retrieveTemplate.template != subtractImages.template"
  - contract: retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).template.name
    msg: "retrieveTemplate.template != sampleSpatialMetrics.template"
  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).difference.name ==
              detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).difference.name
    msg: "subtractImages.difference != detectAndMeasure.difference"
  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
              detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).science.name
    msg: "subtractImages.science != detectAndMeasure.science"
  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).template.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).template.name
    msg: "subtractImages.template != diaPipe.template"
  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
    msg: "subtractImages.science != diaPipe.exposure"
  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).matchedTemplate.name ==
              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).matchedTemplate.name
    msg: "subtractImages.matchedTemplate != sampleSpatialMetrics.matchedTemplate"
  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
              filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).diaSourceCat.name
    msg: "detectAndMeasure.diaSources != filterDiaSrcCat.diaSourceCat"
  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
              rbClassify.connections.ConnectionsClass(config=rbClassify).difference.name
    msg: "detectAndMeasure.subtractedMeasuredExposure != rbClassify.difference"
  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
              transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diffIm.name
    msg: "detectAndMeasure.subtractedMeasuredExposure != transformDiaSrcCat.diffIm"
  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name
    msg: "detectAndMeasure.subtractedMeasuredExposure != diaPipe.diffIm"
  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).difference.name
    msg: "detectAndMeasure.subtractedMeasuredExposure != sampleSpatialMetrics.difference"
  - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
              rbClassify.connections.ConnectionsClass(config=rbClassify).diaSources.name
    msg: "filterDiaSrcCat.filteredDiaSourceCat != rbClassify.diaSources"
  - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
              transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name
    msg: "filterDiaSrcCat.filteredDiaSourceCat != transformDiaSrcCat.diaSourceCat"
  - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).diaSources.name
    msg: "filterDiaSrcCat.filteredDiaSourceCat != sampleSpatialMetrics.diaSources"
  - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).longTrailedSources.name ==
      analyzeTrailedDiaSrcCore.connections.ConnectionsClass(config=analyzeTrailedDiaSrcCore).data.name
    msg: "filterDiaSrcCat.longTrailedSources != analyzeTrailedDiaSrcCore.data"
  - contract: (not transformDiaSrcCat.doIncludeReliability) or
              (rbClassify.connections.ConnectionsClass(config=rbClassify).classifications.name ==
               transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).reliability.name)
    msg: "rbClassify.classifications != transformDiaSrcCat.reliability"
  - contract: transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceTable.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).diaSourceTable.name
    msg: "transformDiaSrcCat.diaSourceTable != diaPipe.diaSourceTable"
  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).stars_footprints.name ==
              getRegionTimeFromVisit.connections.ConnectionsClass(config=getRegionTimeFromVisit).dummy_visit.name
    msg: "calibrateImage.stars_footprints != getRegionTimeFromVisit.dummy_visit"
  - contract: getRegionTimeFromVisit.connections.ConnectionsClass(config=getRegionTimeFromVisit).output.name ==
              mpSkyEphemerisQuery.connections.ConnectionsClass(config=mpSkyEphemerisQuery).predictedRegionTime.name
    msg: "mpSkyEphemerisQuery.predictedRegionTime != getRegionTimeFromVisit.output"    
  - contract: (not diaPipe.doSolarSystemAssociation) or
              (mpSkyEphemerisQuery.connections.ConnectionsClass(config=mpSkyEphemerisQuery).ssObjects.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).solarSystemObjectTable.name)
    msg: "mpSkyEphemerisQuery.ssObjects != diaPipe.solarSystemObjectTable"        
  - contract: diaPipe.connections.ConnectionsClass(config=diaPipe).associatedDiaSources.name ==
              analyzeAssocDiaSrcCore.connections.ConnectionsClass(config=analyzeAssocDiaSrcCore).data.name
    msg: "diaPipe.associatedDiaSources != analyzeAssocDiaSrcCore.data"
  - contract: sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).spatiallySampledMetrics.name ==
              diffimTaskPlots.connections.ConnectionsClass(config=diffimTaskPlots).data.name
    msg: "sampleSpatialMetrics.spatiallySampledMetrics != diffimTaskPlots.data"