ApPipeWithFakes.yaml

description: AP Pipeline with synthetic/fake sources. Templates are inputs.
# This pipeline is imported by camera-specific pipelines.
# You almost certainly want to run one of those, and not this one.

# Per RFC-997, all variants of ApPipe must define three mutually exclusive subsets:
# - preload is tasks that can be run before raw images arrive
# - prompt is everything done by Prompt Processing starting from raws
# - afterburner is metrics and other non-essential tasks that are skipped by Prompt Processing

imports:
  - location: $ANALYSIS_TOOLS_DIR/pipelines/apDetectorVisitQualityCore.yaml
  - location: $ANALYSIS_TOOLS_DIR/pipelines/apDetectorVisitFakesCore.yaml

parameters:
  coaddName: goodSeeing
  # APDB config file must be user-specified
  apdb_config:
  fakesType: 'fakes_'
  injected_prefix: 'fakes_'
  injection_prefix: 'injection_'

tasks:
  isr:
    class: lsst.ip.isr.IsrTask
    config:
      doBrighterFatter: False
  calibrateImage:
    class: lsst.pipe.tasks.calibrateImage.CalibrateImageTask
  inject_visit:
    class: lsst.source.injection.inject_visit.VisitInjectTask
    config:
      external_psf: false
      external_photo_calib: false
      external_wcs: false
      connections.input_exposure: initial_pvi
      connections.output_exposure: fakes_initial_pvi
      connections.output_catalog: fakes_initial_pvi_catalog
      connections.injection_prefix: parameters.injection_prefix
      connections.injected_prefix: parameters.injected_prefix
  loadDiaCatalogs:
    class: lsst.ap.association.LoadDiaCatalogsTask
    config:
      apdb_config_url: parameters.apdb_config
  retrieveTemplate:
    class: lsst.ip.diffim.getTemplate.GetTemplateTask
    config:
      connections.coaddName: parameters.coaddName
      connections.fakesType: parameters.fakesType
      connections.coaddExposures: goodSeeingCoadd
      connections.wcs: initial_pvi.wcs
      connections.bbox: initial_pvi.bbox
  subtractImages:
    class: lsst.ip.diffim.subtractImages.AlardLuptonSubtractTask
    config:
      connections.coaddName: parameters.coaddName
      connections.fakesType: parameters.fakesType
      connections.science: fakes_initial_pvi
      connections.sources: initial_stars_footprints_detector
      doApplyExternalCalibrations: false
  detectAndMeasure:
    class: lsst.ip.diffim.detectAndMeasure.DetectAndMeasureTask
    config:
      connections.coaddName: parameters.coaddName
      connections.fakesType: parameters.fakesType
      connections.science: fakes_initial_pvi
      doSkySources: true
  filterDiaSrcCat:
    class: lsst.ap.association.FilterDiaSourceCatalogTask
    config:
      doRemoveSkySources: true
      connections.coaddName: parameters.coaddName
      connections.fakesType: parameters.fakesType
  rbClassify:
    class: lsst.meas.transiNet.RBTransiNetTask
    config:
      modelPackageStorageMode: butler
      connections.coaddName: parameters.coaddName
      connections.fakesType: parameters.fakesType
      connections.science: fakes_initial_pvi
  transformDiaSrcCat:
    class: lsst.ap.association.TransformDiaSourceCatalogTask
    config:
      connections.coaddName: parameters.coaddName
      connections.fakesType: parameters.fakesType
      doRemoveSkySources: True
      doIncludeReliability: True  # Output from rbClassify
  getRegionTimeFromVisit:
    class: lsst.pipe.tasks.getRegionTimeFromVisit.GetRegionTimeFromVisitTask
    config:
      connections.coaddName: parameters.coaddName
      connections.fakesType: parameters.fakesType
  mpSkyEphemerisQuery:
    class: lsst.ap.association.MPSkyEphemerisQueryTask
  diaPipe:
    class: lsst.ap.association.DiaPipelineTask
    config:
      doWriteAssociatedSources: True
      doSolarSystemAssociation: False
      apdb_config_url: parameters.apdb_config
      doPackageAlerts: True  # Test alert generation, but don't output
      connections.coaddName: parameters.coaddName
      connections.fakesType: parameters.fakesType
      connections.exposure: fakes_initial_pvi
  injectedMatch:
    class: lsst.ap.pipe.matchSourceInjected.MatchInitialPVIInjectedTask
    config:
      connections.coaddName: parameters.coaddName
      connections.fakesType: parameters.fakesType
      connections.injectedInitialPVICat: fakes_initial_pvi_catalog
      matchDistanceArcseconds: 0.5
      trimBuffer: 50
  consolidateMatchDiaSrc:
    class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
    config:
      # Task doesn't support coaddName, so coopt catalogType instead.
      connections.catalogType: parameters.coaddName
      connections.inputCatalogs: "fakes_{catalogType}Diff_matchDiaSrc"
      connections.outputCatalog: "fakes_{catalogType}Diff_matchDiaSourceTable"
  sampleSpatialMetrics:
    class: lsst.ip.diffim.SpatiallySampledMetricsTask
    config:
      connections.coaddName: parameters.coaddName
      connections.fakesType: parameters.fakesType
      connections.science: fakes_initial_pvi
subsets:
  apPipeSingleFrame:
    subset:
      - isr
      - calibrateImage
      - initialPviCore
    description: >-
      The prompt ApPipe tasks that make up single-frame processing. Not to be confused with the
      SingleFrame.yaml pipeline, which does more than just ApPipe single frame processing, and
      is designed as a standalone alternative to ApPipeWithFakes.yaml.
  processCcd:  # TODO: remove on DM-48428
    subset:
      - isr
      - calibrateImage
      - initialPviCore
    description: Deprecated alias for apPipeSingleFrame, will be removed after v29.
  apPipe:
    subset:
      - isr
      - calibrateImage
      - inject_visit
      - loadDiaCatalogs
      - retrieveTemplate
      - subtractImages
      - detectAndMeasure
      - filterDiaSrcCat
      - rbClassify
      - transformDiaSrcCat
      - getRegionTimeFromVisit
      - mpSkyEphemerisQuery
      - diaPipe
      - injectedMatch
      - consolidateMatchDiaSrc
      - sampleSpatialMetrics
      - analyzeAssocDiaSrcCore
      - analyzeTrailedDiaSrcCore
      - analyzeDiaFakesDetectorVisitCore
      - analyzeDiaFakesVisitCore
      - diffimTaskCore
      - detectionTaskCore
      - associationCore
      - associationTiming
      - diffimTaskPlots
      - initialPviCore
    description: >
      An alias of ApPipe to use in higher-level pipelines.
  preload:
    subset:
      - loadDiaCatalogs
      - mpSkyEphemerisQuery
    description: Tasks that can be run before receiving raw images.
  prompt:
    subset:
      - isr
      - calibrateImage
      - inject_visit
      - retrieveTemplate
      - subtractImages
      - detectAndMeasure
      - filterDiaSrcCat
      - rbClassify
      - transformDiaSrcCat
      - diaPipe
      - injectedMatch
      - consolidateMatchDiaSrc
      - analyzeAssocDiaSrcCore
      - analyzeTrailedDiaSrcCore
      - diffimTaskCore
      - detectionTaskCore
      - associationCore
      - associationTiming
      - initialPviCore
    description: >
      Tasks necessary to turn raw images into APDB rows and alerts.
      Requires preload subset to be run first.
  afterburner:
    subset:
      - sampleSpatialMetrics
      - diffimTaskPlots
      - analyzeDiaFakesDetectorVisitCore
      - analyzeDiaFakesVisitCore
    description: >
      Tasks for QA and other non-real-time processing.
      Requires prompt subset to be run first.
  injected_apPipeSingleFrame:
    subset:
    - inject_visit
    description: >-
      All tasks from the 'apPipeSingleFrame' subset impacted by source injection.
  injected_processCcd:
    subset:
    - inject_visit
    description: >
      Deprecated alias for injected_apPipeSingleFrame, will be removed after v29.
  injected_apPipe:
    subset:
    - inject_visit
    - subtractImages
    - detectAndMeasure
    - transformDiaSrcCat
    - rbClassify
    - diaPipe
    - injectedMatch
    - consolidateMatchDiaSrc
    - analyzeDiaFakesDetectorVisitCore
    - analyzeDiaFakesVisitCore
    description: >
      All tasks from the 'apPipe' subset impacted by source injection.
contracts:
  - contract: detectAndMeasure.doSkySources == transformDiaSrcCat.doRemoveSkySources
  # Both loadDiaCatalogs and diaPipe connect to the APDB, so make sure they use the same configuration
  - loadDiaCatalogs.apdb_config_url == diaPipe.apdb_config_url
  # Inputs and outputs must match. For consistency, contracts are written in execution order:
  #     first task == second task, then sorted by (first, second)
  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
  - contract: retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
              subtractImages.connections.ConnectionsClass(config=subtractImages).template.name
    msg: "retrieveTemplate.template != subtractImages.template"
  - contract: retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).template.name
    msg: "retrieveTemplate.template != sampleSpatialMetrics.template"
  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).difference.name ==
              detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).difference.name
    msg: "subtractImages.difference != detectAndMeasure.difference"
  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
              detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).science.name
    msg: "subtractImages.science != detectAndMeasure.science"
  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).template.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).template.name
    msg: "subtractImages.template != diaPipe.template"
  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
    msg: "subtractImages.science != diaPipe.exposure"
  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).matchedTemplate.name ==
              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).matchedTemplate.name
    msg: "subtractImages.matchedTemplate != sampleSpatialMetrics.matchedTemplate"
  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
              filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).diaSourceCat.name
    msg: "detectAndMeasure.diaSources != filterDiaSrcCat.diaSourceCat"
  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
              rbClassify.connections.ConnectionsClass(config=rbClassify).difference.name
    msg: "detectAndMeasure.subtractedMeasuredExposure != rbClassify.difference"
  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
              transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diffIm.name
    msg: "detectAndMeasure.subtractedMeasuredExposure != transformDiaSrcCat.diffIm"
  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name
    msg: "detectAndMeasure.subtractedMeasuredExposure != diaPipe.diffIm"
  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).difference.name
    msg: "detectAndMeasure.subtractedMeasuredExposure != sampleSpatialMetrics.difference"
  - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
              rbClassify.connections.ConnectionsClass(config=rbClassify).diaSources.name
    msg: "filterDiaSrcCat.filteredDiaSourceCat != rbClassify.diaSources"
  - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
              transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name
    msg: "filterDiaSrcCat.filteredDiaSourceCat != transformDiaSrcCat.diaSourceCat"
  - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).diaSources.name
    msg: "filterDiaSrcCat.filteredDiaSourceCat != sampleSpatialMetrics.diaSources"
  - contract: (not transformDiaSrcCat.doIncludeReliability) or
              (rbClassify.connections.ConnectionsClass(config=rbClassify).classifications.name ==
               transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).reliability.name)
    msg: "rbClassify.classifications != transformDiaSrcCat.reliability"
  - contract: transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceTable.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).diaSourceTable.name
    msg: "transformDiaSrcCat.diaSourceTable != diaPipe.diaSourceTable"
  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).stars_footprints.name ==
              getRegionTimeFromVisit.connections.ConnectionsClass(config=getRegionTimeFromVisit).dummy_visit.name
    msg: "calibrateImage.stars_footprints != getRegionTimeFromVisit.dummy_visit"
  - contract: getRegionTimeFromVisit.connections.ConnectionsClass(config=getRegionTimeFromVisit).output.name ==
              mpSkyEphemerisQuery.connections.ConnectionsClass(config=mpSkyEphemerisQuery).predictedRegionTime.name
    msg: "mpSkyEphemerisQuery.predictedRegionTime != getRegionTimeFromVisit.output"
  - contract: (not diaPipe.doSolarSystemAssociation) or
              (mpSkyEphemerisQuery.connections.ConnectionsClass(config=mpSkyEphemerisQuery).ssObjects.name ==
              diaPipe.connections.ConnectionsClass(config=diaPipe).solarSystemObjectTable.name)
    msg: "mpSkyEphemerisQuery.ssObjects != diaPipe.solarSystemObjectTable"
  - contract: diaPipe.connections.ConnectionsClass(config=diaPipe).associatedDiaSources.name ==
              analyzeAssocDiaSrcCore.connections.ConnectionsClass(config=analyzeAssocDiaSrcCore).data.name
    msg: "diaPipe.associatedDiaSources != analyzeAssocDiaSrcCore.data"
  - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).longTrailedSources.name ==
              analyzeTrailedDiaSrcCore.connections.ConnectionsClass(config=analyzeTrailedDiaSrcCore).data.name
    msg: "diaPipe.longTrailedSources != analyzeTrailedDiaSrcCore.data"
  - contract: sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).spatiallySampledMetrics.name ==
              diffimTaskPlots.connections.ConnectionsClass(config=diffimTaskPlots).data.name
    msg: "sampleSpatialMetrics.spatiallySampledMetrics != diffimTaskPlots.data"