Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Be/create ifu wkf #89

Merged
merged 20 commits into from
Feb 10, 2025
Merged

Be/create ifu wkf #89

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
38e13e1
Added the IFU wkf skeleton
eiseleb47 Nov 16, 2024
96a3aa3
Fixed extra colon in classification
eiseleb47 Nov 20, 2024
be99e00
Added badpix map
eiseleb47 Nov 21, 2024
30bfaff
Fixed typo
eiseleb47 Dec 2, 2024
ac5b0b4
Merge remote-tracking branch 'refs/remotes/origin/mb/ifu-missing' int…
Dec 13, 2024
c5482ce
Fix Filters and Typos
Dec 15, 2024
caa9fe9
Fixed task conventions
Jan 5, 2025
ccecde8
Temporary fix for distortion task
Jan 26, 2025
e87b9fb
Merge branch 'main' into be/create_ifu_wkf
eiseleb47 Jan 26, 2025
f80a609
Ignoring Pinhole in Distortion for now
eiseleb47 Jan 27, 2025
aa7162f
Fixed small data source errors
eiseleb47 Jan 29, 2025
7b1e1e2
Fixed Calib matchings
eiseleb47 Jan 31, 2025
8d1b460
Reverted Distortion changes and added ifu conventions
eiseleb47 Feb 1, 2025
49029fa
Added badpix ifu classification, metis_wkf import and meta target
eiseleb47 Feb 2, 2025
ea6ac56
Run full metis_wkf instead of only lm
hugobuddel Feb 2, 2025
8e79e06
Merge branch 'main' into hb/runfullmetiswkf
hugobuddel Feb 2, 2025
68a1a83
Added wcu_off and sky
eiseleb47 Feb 2, 2025
d8410f0
Merge branch 'main' into be/create_ifu_wkf
eiseleb47 Feb 6, 2025
a069c5d
Fixed run_edps.yaml
eiseleb47 Feb 6, 2025
8b5daaa
Telluric recipe fix from baeb560
eiseleb47 Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/run_edps.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ jobs:
export SOF_DATA="$(pwd)/METIS_Pipeline_Test_Data/metis_sim_small_1/data"
export SOF_DIR="$(pwd)/METIS_Pipeline_Test_Data/metis_sim_small_1/sof"
edps -lw
edps -w metis.metis_lm_img_wkf -i $SOF_DATA -c
edps -w metis.metis_lm_img_wkf -i $SOF_DATA -lt
edps -w metis.metis_lm_img_wkf -i $SOF_DATA -m all| tee edps.stdout.txt
edps -w metis.metis_wkf -i $SOF_DATA -c
edps -w metis.metis_wkf -i $SOF_DATA -lt
edps -w metis.metis_wkf -i $SOF_DATA -m all| tee edps.stdout.txt
! grep "'FAILED'" edps.stdout.txt
4 changes: 2 additions & 2 deletions metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_telluric.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ class ProductResponseFunction(PipelineProduct):
Final product: response curve for the flux calibration
"""
level = cpl.ui.Frame.FrameLevel.FINAL
tag = r"IFU_TELLURIC"
tag = r"FLUXCAL_TAB"
frame_type = cpl.ui.Frame.FrameType.IMAGE

# TODO: Define input type for the paramfile in common.py
Expand Down Expand Up @@ -125,7 +125,7 @@ def process_images(self) -> Dict[str, PipelineProduct]:
image = self._create_dummy_image()

self.products = {
product.category: product(self, header, image)
product.tag: product(self, header, image)
for product in [self.ProductTelluricTransmission, self.ProductResponseFunction]
}
return self.products
Expand Down
325 changes: 325 additions & 0 deletions metisp/workflows/metis/metis_lm_ifu_wkf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,325 @@
from edps import task, data_source, classification_rule, SCIENCE

# --- Classification Rules ---

badpix_ifu_class = classification_rule("BADPIX_MAP_IFU",
{"pro.catg": "BADPIX_MAP_det",
})

detlin_ifu_class = classification_rule("DETLIN_IFU_RAW",
{"instrume": "METIS",
"dpr.catg": "CALIB",
"dpr.type": "DETLIN",
"dpr.tech": "IFU",
})

gain_map_ifu_class = classification_rule("GAIN_MAP_IFU",
{"pro.catg": "GAIN_MAP_det",
})

lin_det_ifu_class = classification_rule("LINEARITY_IFU",
{"pro.catg": "LINEARITY_det",
})

rawdark_ifu_class = classification_rule("DARK_IFU_RAW",
{"instrume": "METIS",
"dpr.catg": "CALIB",
"dpr.tech": "IFU",
"dpr.type": "DARK",
})

distortion_ifu_class = classification_rule("IFU_DISTORTION_RAW",
{"instrume": "METIS",
"dpr.catg": "CALIB",
"dpr.tech": "IFU",
"dpr.type": "DISTORTION",
})

wave_ifu_class = classification_rule("IFU_WAVE_RAW",
{"instrume": "METIS",
"dpr.catg": "CALIB",
"dpr.tech": "IFU",
"dpr.type": "WAVE",
})

wave_cal_ifu_class = classification_rule("IFU_WAVECAL",
{"pro.catg": "IFU_WAVECAL",
})

rsrf_ifu_class = classification_rule("IFU_RSRF_RAW",
{"instrume": "METIS",
"dpr.catg": "CALIB",
"dpr.tech": "IFU",
"dpr.type": "RSRF",
})

wcu_off_ifu_class = classification_rule("IFU_WCU_OFF_RAW",
{"instrume": "METIS",
"dpr.catg": "CALIB",
"dpr.tech": "IFU",
"dpr.type": "DARK,WCUOFF",
})

rsrf_prod_ifu_class = classification_rule("RSRF_IFU",
{"pro.catg": "RSRF_IFU",
})

std_ifu_class = classification_rule("IFU_STD_RAW",
{"instrume": "METIS",
"dpr.catg": "CALIB",
"dpr.tech": "IFU",
"dpr.type": "STD",
})

sky_ifu_class = classification_rule("IFU_SKY_RAW",
{"instrume": "METIS",
"dpr.catg": "CALIB",
"dpr.tech": "IFU",
"dpr.type": "SKY",
})

sci_ifu_class = classification_rule("IFU_SCI_RAW",
{"instrume": "METIS",
"dpr.catg": "SCIENCE",
"dpr.tech": "IFU",
"dpr.type": "OBJECT",
})

persistence_class = classification_rule("PERSISTENCE_MAP",
{"pro.catg": "PERSISTENCE_MAP",
})

pinhole_class = classification_rule("PINHOLE_TABLE",
{"pro.catg": "PINHOLE_TABLE",
})

master_dark_ifu_class = classification_rule("MASTER_DARK_IFU",
{"pro.catg": "MASTER_DARK_IFU",
})

distortion_table_ifu_class = classification_rule("IFU_DISTORTION_TABLE",
{"pro.catg": "IFU_DISTORTION_TABLE",
})

calib_rsrf_ifu_class = classification_rule("RSRF_IFU",
{"pro.catg": "RSRF_IFU",
})

sci_comb_ifu_class = classification_rule("IFU_SCI_COMBINED",
{"pro.catg": "IFU_SCI_COMBINED",
})

sci_reduce_ifu_class = classification_rule("IFU_SCI_REDUCED",
{"pro.catg": "IFU_SCI_REDUCED",
})

std_comb_ifu_class = classification_rule("IFU_STD_COMBINED",
{"pro.catg": "IFU_STD_COMBINED",
})

fluxstd_ifu_class = classification_rule("FLUXSTD_CATALOG",
{"pro.catg": "FLUXSTD_CATALOG",
})

lsf_kernel_class = classification_rule("LSF_KERNEL",
{"pro.catg": "LSF_KERNEL",
})

atm_profile_class = classification_rule("ATM_PROFILE",
{"pro.catg": "ATM_PROFILE",
})

telluric_ifu_class =classification_rule("IFU_TELLURIC")

flux_tab_class = classification_rule("FLUXCAL_TAB")

# --- Data sources ---

bad_pix_ifu_calib = (data_source()
.with_classification_rule(badpix_ifu_class)
.build())

detlin_ifu_raw = (data_source()
.with_classification_rule(detlin_ifu_class)
.with_match_keywords(["instrume"])
.build())

dark_ifu_raw = (data_source()
.with_classification_rule(rawdark_ifu_class)
.with_match_keywords(["instrume"])
.build())

distortion_ifu_raw = (data_source()
.with_classification_rule(distortion_ifu_class)
.with_match_keywords(["instrume"])
.build())

wave_ifu_raw = (data_source()
.with_classification_rule(wave_ifu_class)
.with_match_keywords(["instrume"])
.build())

rsrf_ifu_raw = (data_source()
.with_classification_rule(rsrf_ifu_class)
.with_match_keywords(["instrume"])
.build())

wcu_off_ifu_raw = (data_source()
.with_classification_rule(wcu_off_ifu_class)
.with_match_keywords(["instrume"])
.build())

std_ifu_raw = (data_source()
.with_classification_rule(std_ifu_class)
.with_match_keywords(["instrume"])
.build())

sky_ifu_raw = (data_source()
.with_classification_rule(sky_ifu_class)
.with_match_keywords(["instrume"])
.build())

sci_ifu_raw = (data_source()
.with_classification_rule(sci_ifu_class)
.with_match_keywords(["instrume"])
.build())

calib_persistence = (data_source()
.with_classification_rule(persistence_class)
.with_match_keywords(["simple"])
.build())

calib_pinhole = (data_source()
.with_classification_rule(pinhole_class)
.with_match_keywords(["simple"])
.build())

calib_lsf_kernel = (data_source()
.with_classification_rule(lsf_kernel_class)
.with_match_keywords(["simple"])
.build())

calib_flux_std = (data_source()
.with_classification_rule(fluxstd_ifu_class)
.with_match_keywords(["simple"])
.build())

calib_atm_profile = (data_source()
.with_classification_rule(atm_profile_class)
.with_match_keywords(["simple"])
.build())

# --- Processing tasks ---


lingain_ifu_task = (task("metis_ifu_lingain")
.with_recipe("metis_det_lingain")
.with_main_input(detlin_ifu_raw)
.with_associated_input(bad_pix_ifu_calib, min_ret=0)
.build())

dark_ifu_task = (task("metis_ifu_dark")
.with_recipe("metis_det_dark")
.with_main_input(dark_ifu_raw)
.with_associated_input(bad_pix_ifu_calib, min_ret=0)
.with_associated_input(lingain_ifu_task)
.with_associated_input(calib_persistence, min_ret=0)
.with_input_filter(lin_det_ifu_class, gain_map_ifu_class, persistence_class)
.build())

distortion_ifu_task = (task("metis_ifu_distortion")
.with_recipe("metis_ifu_distortion")
.with_main_input(distortion_ifu_raw)
.with_associated_input(bad_pix_ifu_calib, min_ret=0)
.with_associated_input(lingain_ifu_task)
.with_associated_input(calib_persistence, min_ret=0)
.with_associated_input(calib_pinhole)
.with_associated_input(dark_ifu_task)
.with_input_filter(lin_det_ifu_class, gain_map_ifu_class, master_dark_ifu_class, persistence_class, pinhole_class)
.build())

wave_ifu_task = (task("metis_ifu_wavecal")
.with_recipe("metis_ifu_wavecal")
.with_main_input(wave_ifu_raw)
.with_associated_input(bad_pix_ifu_calib, min_ret=0)
.with_associated_input(lingain_ifu_task)
.with_associated_input(calib_persistence, min_ret=0)
.with_associated_input(dark_ifu_task)
.with_associated_input(distortion_ifu_task)
.with_input_filter(lin_det_ifu_class, gain_map_ifu_class, master_dark_ifu_class, distortion_table_ifu_class, persistence_class, pinhole_class)
.build())

rsrf_ifu_task = (task("metis_ifu_rsrf")
.with_recipe("metis_ifu_rsrf")
.with_main_input(rsrf_ifu_raw)
.with_associated_input(bad_pix_ifu_calib, min_ret=0)
.with_associated_input(lingain_ifu_task)
.with_associated_input(calib_persistence, min_ret=0)
.with_associated_input(dark_ifu_task)
.with_associated_input(distortion_ifu_task)
.with_associated_input(wcu_off_ifu_raw)
.with_associated_input(wave_ifu_task)
.with_input_filter(lin_det_ifu_class, gain_map_ifu_class, master_dark_ifu_class, distortion_table_ifu_class, wave_cal_ifu_class, persistence_class)
.build())

std_ifu_task = (task("metis_ifu_std_reduce")
.with_recipe("metis_ifu_reduce")
.with_main_input(std_ifu_raw)
.with_associated_input(bad_pix_ifu_calib, min_ret=0)
.with_associated_input(lingain_ifu_task)
.with_associated_input(calib_persistence, min_ret=0)
.with_associated_input(dark_ifu_task)
.with_associated_input(distortion_ifu_task)
.with_associated_input(wave_ifu_task)
.with_associated_input(rsrf_ifu_task)
.with_associated_input(sky_ifu_raw)
.with_input_filter(lin_det_ifu_class, gain_map_ifu_class, master_dark_ifu_class, persistence_class, distortion_table_ifu_class, wave_cal_ifu_class, rsrf_prod_ifu_class)
.build())

sci_ifu_task = (task("metis_ifu_sci_reduce")
.with_recipe("metis_ifu_reduce")
.with_main_input(sci_ifu_raw)
.with_associated_input(bad_pix_ifu_calib, min_ret=0)
.with_associated_input(lingain_ifu_task)
.with_associated_input(calib_persistence, min_ret=0)
.with_associated_input(dark_ifu_task)
.with_associated_input(distortion_ifu_task)
.with_associated_input(wave_ifu_task)
.with_associated_input(rsrf_ifu_task)
.with_associated_input(sky_ifu_raw)
.with_input_filter(lin_det_ifu_class, gain_map_ifu_class, master_dark_ifu_class, persistence_class, distortion_table_ifu_class, wave_cal_ifu_class, rsrf_prod_ifu_class)
hugobuddel marked this conversation as resolved.
Show resolved Hide resolved
.build())

telluric_sci_ifu_task = (task("metis_ifu_sci_telluric")
.with_recipe("metis_ifu_telluric")
.with_main_input(sci_ifu_task)
.with_associated_input(calib_flux_std)
.with_associated_input(calib_lsf_kernel)
.with_associated_input(calib_atm_profile)
.with_input_filter(sci_reduce_ifu_class, sci_comb_ifu_class, fluxstd_ifu_class, lsf_kernel_class, atm_profile_class)
.with_output_filter(telluric_ifu_class)
.build())

telluric_std_ifu_task = (task("metis_ifu_std_telluric")
.with_recipe("metis_ifu_telluric")
.with_main_input(std_ifu_task)
.with_associated_input(calib_flux_std)
.with_associated_input(calib_lsf_kernel)
.with_associated_input(calib_atm_profile)
.with_input_filter(std_comb_ifu_class, fluxstd_ifu_class, lsf_kernel_class, atm_profile_class)
.with_output_filter(flux_tab_class)
.build())

calibrate_ifu_task = (task("metis_ifu_calibrate")
.with_recipe("metis_ifu_calibrate")
.with_main_input(sci_ifu_task)
.with_associated_input(telluric_sci_ifu_task)
.with_associated_input(telluric_std_ifu_task)
.with_input_filter(sci_reduce_ifu_class, flux_tab_class, telluric_ifu_class)
.build())

post_process_ifu_task = (task("metis_ifu_postprocess")
.with_recipe("metis_ifu_postprocess")
.with_main_input(calibrate_ifu_task)
.with_meta_targets([SCIENCE])
.build())
2 changes: 1 addition & 1 deletion metisp/workflows/metis/metis_wkf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@

from .metis_datasources import *
from .metis_lm_img_wkf import *

from .metis_lm_ifu_wkf import *