Merge pull request #69 from AstarVienna/hb/checksofdrld

Check whether SOF files are consistent with DRLD
AstarVienna · Nov 13, 2024 · 926b1b7 · 926b1b7
2 parents 97d4111 + f297a8c
commit 926b1b7
Show file tree

Hide file tree

Showing 100 changed files with 287 additions and 191 deletions.
diff --git a/ESO/check_sof_files.py b/ESO/check_sof_files.py
@@ -1,33 +1,137 @@
 #!/usr/bin/env python
+"""Check SOF files.
 
+First install the DRLD with
+METIS_DRLD> pip install -e .
+
+But that does not always work (TODO: fix), so you can also specify the
+Python path:
+export PYTHONPATH=/path/to/METIS_DRLD/codes
+"""
 from pathlib import Path
+from codes.drld_parser.data_reduction_library_design import METIS_DataReductionLibraryDesign
+
 
 PATH_HERE = Path(__file__).parent
-PATH_SOFS = PATH_HERE / "sofFiles"
 PATH_FITS = PATH_HERE / "output"
 
-problems = []
 
-for filename in PATH_SOFS.glob("*.sof"):
-    # print(filename)
-    lines = open(filename, mode="r", encoding="utf-8").readlines()
+def check_sof_file(filename, problems_raws, problems_tags, problems_names, problems_input):
+    # E.g. "metis_lm_img_flat.twilight.sof" -> "metis_lm_img_flat"
+    recipe_name = filename.stem.split(".")[0]
+    assert recipe_name in METIS_DataReductionLibraryDesign.recipes, (f"{recipe_name} not found in "
+                                                                     f"METIS_DataReductionLibraryDesign")
+    recipe = METIS_DataReductionLibraryDesign.recipes[recipe_name]
+
+    lines1 = open(filename, mode="r", encoding="utf-8").readlines()
+    lines = [
+        [a.split("/")[-1] for a in line.split()]
+        for line in lines1
+    ]
+
+    # Check whether tags are in the DRLD.
+    tags = [tag for _, tag in lines]
+    tags_missing = [
+        tag for tag in tags
+        if tag not in METIS_DataReductionLibraryDesign.dataitems
+    ]
+    if tags_missing:
+        problems_tags.append((filename.name, tags_missing))
+
+    # Check whether the tag is in the filename.
+    filenames_without_tags = [
+        fn
+        for fn, tag in lines
+        if tag not in fn
+    ]
+    if filenames_without_tags:
+        problems_names.append((filename.name, filenames_without_tags))
+
+    # Check whether raws exist.
     fns_raw = [
-        line.split()[0].split("/")[1]
-        for line in lines
-        if "RAW" in line
+        fn
+        for fn, tag in lines
+        if "RAW" in tag
     ]
     fns_raw_missing = [
         fn
         for fn in fns_raw
         if not (PATH_FITS / fn).exists()
     ]
     if fns_raw_missing:
-        problems.append((filename.name, fns_raw_missing))
-
-if problems:
-    print("Some RAW files are missing:")
-    for fn_sof, fns_missing in problems:
-        print("-", fn_sof)
-        for fn in fns_missing:
-            print("  -", fn)
-    exit(1)
+        problems_raws.append((filename.name, fns_raw_missing))
+
+    # Check whether files mentioned in SOF files are actually input to the recipes.
+    tags_input_to_recipe = [di.name for di in recipe.input_data]
+    tags_not_input_to_recipe_according_to_drld = [
+        tag for tag in tags
+        if tag not in tags_input_to_recipe
+    ]
+    if tags_not_input_to_recipe_according_to_drld:
+        problems_input.append((filename.name, tags_not_input_to_recipe_according_to_drld))
+
+
+def check_sof_directory(directory, check_files_existing=True):
+    problems_raws = []
+    problems_tags = []
+    problems_names = []
+    problems_input = []
+
+    for filename in directory.glob("*.sof"):
+        if "persistence" in filename.name:
+            continue
+        check_sof_file(filename, problems_raws, problems_tags, problems_names, problems_input)
+
+    if problems_raws and check_files_existing:
+        print("Some RAW files are missing:")
+        for fn_sof, fns_missing in problems_raws:
+            print("-", fn_sof)
+            for fn in fns_missing:
+                print("  -", fn)
+
+    print()
+    if problems_tags:
+        print("Some tags are not in the DRLD:")
+        for fn_sof, fns_missing in problems_tags:
+            print("-", fn_sof)
+            for fn in fns_missing:
+                print("  -", fn)
+
+    print()
+    if problems_names:
+        print("Some filenames do not contain the tag:")
+        for fn_sof, fns_missing in problems_names:
+            print("-", fn_sof)
+            for fn in fns_missing:
+                print("  -", fn)
+
+    print()
+    if problems_input:
+        print("Some tags in SOF files are not actually input to the recipe:")
+        for fn_sof, tags_not_input in problems_input:
+            print("-", fn_sof)
+            for fn in tags_not_input:
+                print("  -", fn)
+
+    if problems_raws or problems_tags:
+        return False
+    return True
+
+
+def main():
+    PATH_SOFS = PATH_HERE / "sofFiles"
+    PATH_TEMPLATES = PATH_HERE / "sofTemplates"
+
+    print("Doing the SOF files.")
+    sofs_ok = check_sof_directory(PATH_SOFS)
+
+    print()
+    print("Doing the Templates.")
+    templates_ok = check_sof_directory(PATH_TEMPLATES, check_files_existing=False)
+
+    return sofs_ok and templates_ok
+
+if __name__ == "__main__":
+    all_ok = main()
+    if not all_ok:
+        exit(1)
diff --git a/ESO/sofFiles/metis_lm_lss_mf_calctrans.sof → ESO/sofFiles/metis_LM_lss_mf_calctrans.sof b/ESO/sofFiles/metis_lm_lss_mf_calctrans.sof → ESO/sofFiles/metis_LM_lss_mf_calctrans.sof
diff --git a/ESO/sofFiles/metis_lm_lss_mf_correct.sof → ESO/sofFiles/metis_LM_lss_mf_correct.sof b/ESO/sofFiles/metis_lm_lss_mf_correct.sof → ESO/sofFiles/metis_LM_lss_mf_correct.sof
diff --git a/ESO/sofFiles/metis_lm_lss_mf_model.sof → ESO/sofFiles/metis_LM_lss_mf_model.sof b/ESO/sofFiles/metis_lm_lss_mf_model.sof → ESO/sofFiles/metis_LM_lss_mf_model.sof
diff --git a/ESO/sofFiles/metis_LM_lss_rsrf.sof b/ESO/sofFiles/metis_LM_lss_rsrf.sof
@@ -0,0 +1,5 @@
+$SOF_DATA/METIS.LM_LSS_RSRF_RAW.2024-01-02_00_11_15.fits LM_LSS_RSRF_RAW
+$SOF_DATA/MASTER_DARK_2RG.fits MASTER_DARK_2RG
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
+$SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
diff --git a/ESO/sofFiles/metis_lm_lss_sci.sof → ESO/sofFiles/metis_LM_lss_sci.sof b/ESO/sofFiles/metis_lm_lss_sci.sof → ESO/sofFiles/metis_LM_lss_sci.sof
@@ -1,12 +1,12 @@
 $SOF_DATA/METIS.LM_LSS_SCI_RAW.2024-01-02_00_24_21.fits LM_LSS_SCI_RAW
-$SOF_DATA/MASTER_DARK_LM.fits MASTER_DARK_2RG
+$SOF_DATA/MASTER_DARK_2RG.fits MASTER_DARK_2RG
 $SOF_DATA/MASTER_LM_LSS_RSRF.fits MASTER_LM_LSS_RSRF
 $SOF_DATA/MASTER_LM_RESPONSE.fits MASTER_LM_RESPONSE
-$SOF_DATA/LINEARITY_LM.fits LINEARITY_2RG
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_LM.fits GAIN_MAP_2RG
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
 $SOF_DATA/LM_LSS_WAVE_GUESS.fits LM_LSS_WAVE_GUESS
-$SOF_DATA/LMM_LSS_DIST_SOL.fits LMM_LSS_DIST_SOL
+$SOF_DATA/LM_LSS_DIST_SOL.fits LM_LSS_DIST_SOL
 $SOF_DATA/ATM_LINE_CAT.fits ATM_LINE_CAT
 $SOF_DATA/LM_ADC_SLITLOSS.fits LM_ADC_SLITLOSS
 $SOF_DATA/STD_TRANSMISSION.fits STD_TRANSMISSION
diff --git a/ESO/sofFiles/metis_lm_lss_std.sof → ESO/sofFiles/metis_LM_lss_std.sof b/ESO/sofFiles/metis_lm_lss_std.sof → ESO/sofFiles/metis_LM_lss_std.sof
@@ -1,11 +1,11 @@
 $SOF_DATA/METIS.LM_LSS_STD_RAW.2024-01-02_00_12_03.fits LM_LSS_STD_RAW
-$SOF_DATA/MASTER_DARK_LM.fits MASTER_DARK_2RG
+$SOF_DATA/MASTER_DARK_2RG.fits MASTER_DARK_2RG
 $SOF_DATA/MASTER_LM_LSS_RSRF.fits MASTER_LM_LSS_RSRF
-$SOF_DATA/LINEARITY_LM.fits LINEARITY_2RG
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_LM.fits GAIN_MAP_2RG
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
 $SOF_DATA/LM_LSS_WAVE_GUESS.fits LM_LSS_WAVE_GUESS
-$SOF_DATA/LMM_LSS_DIST_SOL.fits LMM_LSS_DIST_SOL
+$SOF_DATA/LM_LSS_DIST_SOL.fits LM_LSS_DIST_SOL
 $SOF_DATA/AO_PSF_MODEL.fits AO_PSF_MODEL
 $SOF_DATA/ATM_LINE_CAT.fits ATM_LINE_CAT
 $SOF_DATA/LM_ADC_SLITLOSS.fits LM_ADC_SLITLOSS

diff --git a/ESO/sofFiles/metis_lm_lss_trace.sof → ESO/sofFiles/metis_LM_lss_trace.sof b/ESO/sofFiles/metis_lm_lss_trace.sof → ESO/sofFiles/metis_LM_lss_trace.sof
@@ -1,6 +1,5 @@
 $SOF_DATA/METIS.LM_LSS_RSRF_PINH_RAW.2024-01-02_00_11_37.fits LM_LSS_RSRF_PINH_RAW
 $SOF_DATA/MASTER_LM_LSS_RSRF.fits MASTER_LM_LSS_RSRF
-$SOF_DATA/MASTER_IMG_FLAT_LAMP_LM.fits MASTER_IMG_FLAT_LAMP_LM
-$SOF_DATA/LINEARITY_LM.fits LINEARITY_2RG
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_LM.fits GAIN_MAP_2RG
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
diff --git a/ESO/sofFiles/metis_LM_lss_wave.sof b/ESO/sofFiles/metis_LM_lss_wave.sof
@@ -0,0 +1,7 @@
+$SOF_DATA/METIS.LM_LSS_WAVE_RAW.2024-01-02_00_42_39.fits LM_LSS_WAVE_RAW
+$SOF_DATA/MASTER_DARK_2RG.fits MASTER_DARK_2RG
+$SOF_DATA/MASTER_LM_LSS_RSRF.fits MASTER_LM_LSS_RSRF
+$SOF_DATA/LM_LSS_TRACE.fits LM_LSS_TRACE
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
+$SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
diff --git a/ESO/sofFiles/metis_n_lss_mf_calctrans.sof → ESO/sofFiles/metis_N_lss_mf_calctrans.sof b/ESO/sofFiles/metis_n_lss_mf_calctrans.sof → ESO/sofFiles/metis_N_lss_mf_calctrans.sof
diff --git a/ESO/sofFiles/metis_n_lss_mf_correct.sof → ESO/sofFiles/metis_N_lss_mf_correct.sof b/ESO/sofFiles/metis_n_lss_mf_correct.sof → ESO/sofFiles/metis_N_lss_mf_correct.sof
diff --git a/ESO/sofFiles/metis_n_lss_mf_model.sof → ESO/sofFiles/metis_N_lss_mf_model.sof b/ESO/sofFiles/metis_n_lss_mf_model.sof → ESO/sofFiles/metis_N_lss_mf_model.sof
diff --git a/ESO/sofFiles/metis_N_lss_rsrf.sof b/ESO/sofFiles/metis_N_lss_rsrf.sof
@@ -0,0 +1,5 @@
+$SOF_DATA/METIS.N_LSS_RSRF_RAW.2024-01-02_00_11_17.fits N_LSS_RSRF_RAW
+$SOF_DATA/MASTER_DARK_GEO.fits MASTER_DARK_GEO
+$SOF_DATA/LINEARITY_GEO.fits LINEARITY_GEO
+$SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
+$SOF_DATA/GAIN_MAP_GEO.fits GAIN_MAP_GEO
diff --git a/ESO/sofFiles/metis_n_lss_sci.sof → ESO/sofFiles/metis_N_lss_sci.sof b/ESO/sofFiles/metis_n_lss_sci.sof → ESO/sofFiles/metis_N_lss_sci.sof
@@ -1,12 +1,12 @@
 $SOF_DATA/METIS.N_LSS_SCI_RAW.2024-01-02_00_24_18.fits N_LSS_SCI_RAW
-$SOF_DATA/MASTER_DARK_N.fits MASTER_DARK_GEO
+$SOF_DATA/MASTER_DARK_GEO.fits MASTER_DARK_GEO
 $SOF_DATA/MASTER_N_LSS_RSRF.fits MASTER_N_LSS_RSRF
 $SOF_DATA/MASTER_N_RESPONSE.fits MASTER_N_RESPONSE
-$SOF_DATA/LINEARITY_N.fits LINEARITY_GEO
+$SOF_DATA/LINEARITY_GEO.fits LINEARITY_GEO
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_N.fits GAIN_MAP_GEO
+$SOF_DATA/GAIN_MAP_GEO.fits GAIN_MAP_GEO
 $SOF_DATA/N_LSS_WAVE_GUESS.fits N_LSS_WAVE_GUESS
-$SOF_DATA/NM_LSS_DIST_SOL.fits NM_LSS_DIST_SOL
+$SOF_DATA/N_LSS_DIST_SOL.fits N_LSS_DIST_SOL
 $SOF_DATA/ATM_LINE_CAT.fits ATM_LINE_CAT
 $SOF_DATA/N_ADC_SLITLOSS.fits N_ADC_SLITLOSS
 $SOF_DATA/STD_TRANSMISSION.fits STD_TRANSMISSION
diff --git a/ESO/sofFiles/metis_n_lss_std.sof → ESO/sofFiles/metis_N_lss_std.sof b/ESO/sofFiles/metis_n_lss_std.sof → ESO/sofFiles/metis_N_lss_std.sof
@@ -1,11 +1,11 @@
 $SOF_DATA/METIS.N_LSS_STD_RAW.2024-01-02_00_12_05.fits N_LSS_STD_RAW
-$SOF_DATA/MASTER_DARK_N.fits MASTER_DARK_GEO
+$SOF_DATA/MASTER_DARK_GEO.fits MASTER_DARK_GEO
 $SOF_DATA/MASTER_N_LSS_RSRF.fits MASTER_N_LSS_RSRF
-$SOF_DATA/LINEARITY_N.fits LINEARITY_GEO
+$SOF_DATA/LINEARITY_GEO.fits LINEARITY_GEO
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_N.fits GAIN_MAP_GEO
+$SOF_DATA/GAIN_MAP_GEO.fits GAIN_MAP_GEO
 $SOF_DATA/N_LSS_WAVE_GUESS.fits N_LSS_WAVE_GUESS
-$SOF_DATA/NM_LSS_DIST_SOL.fits NM_LSS_DIST_SOL
+$SOF_DATA/N_LSS_DIST_SOL.fits N_LSS_DIST_SOL
 $SOF_DATA/AO_PSF_MODEL.fits AO_PSF_MODEL
 $SOF_DATA/ATM_LINE_CAT.fits ATM_LINE_CAT
 $SOF_DATA/N_ADC_SLITLOSS.fits N_ADC_SLITLOSS

diff --git a/ESO/sofFiles/metis_n_lss_trace.sof → ESO/sofFiles/metis_N_lss_trace.sof b/ESO/sofFiles/metis_n_lss_trace.sof → ESO/sofFiles/metis_N_lss_trace.sof
@@ -1,6 +1,5 @@
 $SOF_DATA/METIS.N_LSS_RSRF_PINH_RAW.2024-01-02_00_11_38.fits N_LSS_RSRF_PINH_RAW
 $SOF_DATA/MASTER_N_LSS_RSRF.fits MASTER_N_LSS_RSRF
-$SOF_DATA/MASTER_IMG_FLAT_LAMP_N.fits MASTER_IMG_FLAT_LAMP_N
-$SOF_DATA/LINEARITY_N.fits LINEARITY_GEO
+$SOF_DATA/LINEARITY_GEO.fits LINEARITY_GEO
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_N.fits GAIN_MAP_GEO
+$SOF_DATA/GAIN_MAP_GEO.fits GAIN_MAP_GEO
diff --git a/ESO/sofFiles/metis_ifu_dark.sof → ESO/sofFiles/metis_det_dark.ifu.sof b/ESO/sofFiles/metis_ifu_dark.sof → ESO/sofFiles/metis_det_dark.ifu.sof
diff --git a/ESO/sofFiles/metis_lm_dark.sof → ESO/sofFiles/metis_det_dark.lm.sof b/ESO/sofFiles/metis_lm_dark.sof → ESO/sofFiles/metis_det_dark.lm.sof
@@ -1,4 +1,4 @@
 $SOF_DATA/METIS.DARK_2RG_RAW.2024-01-02_00_43_27.fits DARK_2RG_RAW
-$SOF_DATA/LINEARITY_LM.fits LINEARITY_2RG
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_LM.fits GAIN_MAP_2RG
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
diff --git a/ESO/sofFiles/metis_n_dark.sof → ESO/sofFiles/metis_det_dark.n.sof b/ESO/sofFiles/metis_n_dark.sof → ESO/sofFiles/metis_det_dark.n.sof
@@ -1,4 +1,4 @@
 $SOF_DATA/METIS.DARK_GEO_RAW.2024-01-02_00_43_48.fits DARK_GEO_RAW
-$SOF_DATA/LINEARITY_N.fits LINEARITY_GEO
+$SOF_DATA/LINEARITY_GEO.fits LINEARITY_GEO
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_N.fits GAIN_MAP_GEO
+$SOF_DATA/GAIN_MAP_GEO.fits GAIN_MAP_GEO
diff --git a/ESO/sofFiles/metis_ifu_lingain.sof → ESO/sofFiles/metis_det_lingain.ifu.sof b/ESO/sofFiles/metis_ifu_lingain.sof → ESO/sofFiles/metis_det_lingain.ifu.sof
diff --git a/ESO/sofFiles/metis_lm_lingain.sof → ESO/sofFiles/metis_det_lingain.lm.sof b/ESO/sofFiles/metis_lm_lingain.sof → ESO/sofFiles/metis_det_lingain.lm.sof
diff --git a/ESO/sofFiles/metis_n_lingain.sof → ESO/sofFiles/metis_det_lingain.n.sof b/ESO/sofFiles/metis_n_lingain.sof → ESO/sofFiles/metis_det_lingain.n.sof
diff --git a/ESO/sofFiles/metis_ifu_persistence.sof b/ESO/sofFiles/metis_ifu_persistence.sof
diff --git a/ESO/sofFiles/metis_ifu_sci_reduce.sof → ESO/sofFiles/metis_ifu_reduce.sci.sof b/ESO/sofFiles/metis_ifu_sci_reduce.sof → ESO/sofFiles/metis_ifu_reduce.sci.sof
diff --git a/ESO/sofFiles/metis_ifu_std_reduce.sof → ESO/sofFiles/metis_ifu_reduce.std.sof b/ESO/sofFiles/metis_ifu_std_reduce.sof → ESO/sofFiles/metis_ifu_reduce.std.sof
diff --git a/ESO/sofFiles/metis_ifu_sci_telluric.sof → ESO/sofFiles/metis_ifu_telluric.sci.sof b/ESO/sofFiles/metis_ifu_sci_telluric.sof → ESO/sofFiles/metis_ifu_telluric.sci.sof
diff --git a/ESO/sofFiles/metis_ifu_std_telluric.sof → ESO/sofFiles/metis_ifu_telluric.std.sof b/ESO/sofFiles/metis_ifu_std_telluric.sof → ESO/sofFiles/metis_ifu_telluric.std.sof
diff --git a/ESO/sofFiles/metis_ifu_wavecal.sof b/ESO/sofFiles/metis_ifu_wavecal.sof
@@ -1,4 +1,4 @@
-$SOF_DATA/METIS.IFU_WAVE_RAW.2024-01-02_00_42_41.fits *IFU_WAVE_RAW*
+$SOF_DATA/METIS.IFU_WAVE_RAW.2024-01-02_00_42_41.fits IFU_WAVE_RAW
 $SOF_DATA/LINEARITY_IFU.fits LINEARITY_IFU
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
 $SOF_DATA/GAIN_MAP_IFU.fits GAIN_MAP_IFU

diff --git a/ESO/sofFiles/metis_img_lm_chophome.sof → ESO/sofFiles/metis_img_chophome.sof b/ESO/sofFiles/metis_img_lm_chophome.sof → ESO/sofFiles/metis_img_chophome.sof
@@ -1,6 +1,6 @@
 $SOF_DATA/METIS.LM_CHOPPERHOME_RAW.2024-01-02_00_42_42.fits LM_CHOPPERHOME_RAW
-$SOF_DATA/MASTER_DARK_LM.fits MASTER_DARK_2RG
+$SOF_DATA/MASTER_DARK_2RG.fits MASTER_DARK_2RG
 $SOF_DATA/MASTER_IMG_FLAT_LAMP_LM.fits MASTER_IMG_FLAT_LAMP_LM
-$SOF_DATA/LINEARITY_LM.fits LINEARITY_2RG
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_LM.fits GAIN_MAP_2RG
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
diff --git a/ESO/sofFiles/metis_lm_adc_slitloss.sof b/ESO/sofFiles/metis_lm_adc_slitloss.sof
@@ -1,8 +1,8 @@
 $SOF_DATA/METIS.LM_SLITLOSSES_RAW.2024-01-02_00_42_37.fits LM_SLITLOSSES_RAW
 $SOF_DATA/METIS.LM_WCU_OFF_RAW.2024-01-02_00_50_30.fits LM_WCU_OFF_RAW
 $SOF_DATA/METIS.LM_WCU_OFF_RAW.2024-01-02_00_50_31.fits LM_WCU_OFF_RAW
-$SOF_DATA/MASTER_DARK_LM.fits MASTER_DARK_2RG
+$SOF_DATA/MASTER_DARK_2RG.fits MASTER_DARK_2RG
 $SOF_DATA/MASTER_IMG_FLAT_LAMP_LM.fits MASTER_IMG_FLAT_LAMP_LM
-$SOF_DATA/LINEARITY_LM.fits LINEARITY_2RG
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_LM.fits GAIN_MAP_2RG
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
diff --git a/ESO/sofFiles/metis_lm_adi_app.sof b/ESO/sofFiles/metis_lm_adi_app.sof
@@ -1,4 +1,4 @@
 $SOF_DATA/METIS.LM_OFF_AXIS_PSF_RAW.2024-01-02_00_24_26.fits LM_OFF_AXIS_PSF_RAW
 $SOF_DATA/LM_SCI_CALIBRATED.fits LM_SCI_CALIBRATED
 $SOF_DATA/LM_DISTORTION_TABLE.fits LM_DISTORTION_TABLE
-$SOF_DATA/LM_OFN_AXIS_PSF_TEMPLATE.fits LM_OFN_AXIS_PSF_TEMPLATE
+$SOF_DATA/LM_ON_AXIS_PSF_TEMPLATE.fits LM_ON_AXIS_PSF_TEMPLATE
diff --git a/ESO/sofFiles/metis_lm_img_basic_reduce.sof b/ESO/sofFiles/metis_lm_img_basic_reduce.sof
@@ -1,6 +1,6 @@
 $SOF_DATA/METIS.LM_IMAGE_SCI_RAW.2024-01-02_00_30_30.fits LM_IMAGE_SCI_RAW
-$SOF_DATA/MASTER_DARK_LM.fits MASTER_DARK_2RG
-$SOF_DATA/MASTER_IMAGE_FLAT_LAMP_LM.fits MASTER_IMAGE_FLAT_LAMP_LM
-$SOF_DATA/LINEARITY_LM.fits LINEARITY_2RG
+$SOF_DATA/MASTER_DARK_2RG.fits MASTER_DARK_2RG
+$SOF_DATA/MASTER_IMG_FLAT_LAMP_LM.fits MASTER_IMG_FLAT_LAMP_LM
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_LM.fits GAIN_MAP_2RG
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
diff --git a/ESO/sofFiles/metis_lm_img_distortion.sof b/ESO/sofFiles/metis_lm_img_distortion.sof
@@ -1,7 +1,7 @@
 $SOF_DATA/METIS.LM_DISTORTION_RAW.2024-01-02_00_43_21.fits LM_DISTORTION_RAW
 $SOF_DATA/METIS.LM_WCU_OFF_RAW.2024-01-02_00_50_30.fits LM_WCU_OFF_RAW
 $SOF_DATA/METIS.LM_WCU_OFF_RAW.2024-01-02_00_50_31.fits LM_WCU_OFF_RAW
-$SOF_DATA/LINEARITY_LM.fits LINEARITY_2RG
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_LM.fits GAIN_MAP_2RG
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
 $SOF_DATA/PINHOLE_TABLE.fits PINHOLE_TABLE
diff --git a/ESO/sofFiles/metis_lm_img_flat.lamp.sof b/ESO/sofFiles/metis_lm_img_flat.lamp.sof
@@ -0,0 +1,5 @@
+$SOF_DATA/METIS.LM_FLAT_LAMP_RAW.2024-01-02_00_50_46.fits LM_FLAT_LAMP_RAW
+$SOF_DATA/MASTER_DARK_2RG.fits MASTER_DARK_2RG
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
+$SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
diff --git a/ESO/sofFiles/metis_lm_img_flat.twilight.sof b/ESO/sofFiles/metis_lm_img_flat.twilight.sof
@@ -0,0 +1,5 @@
+$SOF_DATA/METIS.LM_FLAT_TWILIGHT_RAW.2024-01-02_00_50_38.fits LM_FLAT_TWILIGHT_RAW
+$SOF_DATA/MASTER_DARK_2RG.fits MASTER_DARK_2RG
+$SOF_DATA/LINEARITY_2RG.fits LINEARITY_2RG
+$SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
+$SOF_DATA/GAIN_MAP_2RG.fits GAIN_MAP_2RG
diff --git a/ESO/sofFiles/metis_lm_img_flat_lamp.sof b/ESO/sofFiles/metis_lm_img_flat_lamp.sof
diff --git a/ESO/sofFiles/metis_lm_img_flat_twilight.sof b/ESO/sofFiles/metis_lm_img_flat_twilight.sof
diff --git a/ESO/sofFiles/metis_lm_lss_rsrf.sof b/ESO/sofFiles/metis_lm_lss_rsrf.sof
diff --git a/ESO/sofFiles/metis_lm_lss_wave.sof b/ESO/sofFiles/metis_lm_lss_wave.sof
diff --git a/ESO/sofFiles/metis_lm_persistence.sof b/ESO/sofFiles/metis_lm_persistence.sof
diff --git a/ESO/sofFiles/metis_n_adc_slitloss.sof b/ESO/sofFiles/metis_n_adc_slitloss.sof
@@ -1,8 +1,8 @@
 $SOF_DATA/METIS.N_SLITLOSSES_RAW.2024-01-02_00_42_38.fits N_SLITLOSSES_RAW
 $SOF_DATA/METIS.N_WCU_OFF_RAW.2024-01-02_00_50_35.fits N_WCU_OFF_RAW
 $SOF_DATA/METIS.N_WCU_OFF_RAW.2024-01-02_00_50_34.fits N_WCU_OFF_RAW
-$SOF_DATA/MASTER_DARK_N.fits MASTER_DARK_GEO
+$SOF_DATA/MASTER_DARK_GEO.fits MASTER_DARK_GEO
 $SOF_DATA/MASTER_IMG_FLAT_LAMP_N.fits MASTER_IMG_FLAT_LAMP_N
-$SOF_DATA/LINEARITY_N.fits LINEARITY_GEO
+$SOF_DATA/LINEARITY_GEO.fits LINEARITY_GEO
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_N.fits GAIN_MAP_GEO
+$SOF_DATA/GAIN_MAP_GEO.fits GAIN_MAP_GEO
diff --git a/ESO/sofFiles/metis_n_img_chopnod.sof b/ESO/sofFiles/metis_n_img_chopnod.sof
@@ -1,6 +1,6 @@
 $SOF_DATA/METIS.N_IMAGE_SCI_RAW.2024-01-02_00_24_14.fits N_IMAGE_SCI_RAW
-$SOF_DATA/MASTER_DARK_N.fits MASTER_DARK_GEO
+$SOF_DATA/MASTER_DARK_GEO.fits MASTER_DARK_GEO
 $SOF_DATA/MASTER_IMG_FLAT_LAMP_N.fits MASTER_IMG_FLAT_LAMP_N
-$SOF_DATA/LINEARITY_N.fits LINEARITY_GEO
+$SOF_DATA/LINEARITY_GEO.fits LINEARITY_GEO
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_N.fits GAIN_MAP_GEO
+$SOF_DATA/GAIN_MAP_GEO.fits GAIN_MAP_GEO
diff --git a/ESO/sofFiles/metis_n_img_distortion.sof b/ESO/sofFiles/metis_n_img_distortion.sof
@@ -1,7 +1,7 @@
 $SOF_DATA/METIS.N_DISTORTION_RAW.2024-01-02_00_43_23.fits N_DISTORTION_RAW
 $SOF_DATA/METIS.N_WCU_OFF_RAW.2024-01-02_00_50_35.fits N_WCU_OFF_RAW
 $SOF_DATA/METIS.N_WCU_OFF_RAW.2024-01-02_00_50_34.fits N_WCU_OFF_RAW
-$SOF_DATA/LINEARITY_N.fits LINEARITY_GEO
+$SOF_DATA/LINEARITY_GEO.fits LINEARITY_GEO
 $SOF_DATA/PERSISTENCE_MAP.fits PERSISTENCE_MAP
-$SOF_DATA/GAIN_MAP_N.fits GAIN_MAP_GEO
+$SOF_DATA/GAIN_MAP_GEO.fits GAIN_MAP_GEO
 $SOF_DATA/PINHOLE_TABLE.fits PINHOLE_TABLE