Merge pull request #789 from desihub/ADM-mtl-maintain

A couple of MTL maintenance updates

bin/run_mtl_loop

@@ -12,10 +12,10 @@ ap = ArgumentParser(description='Make an initial HEALPixel-split ledger for a Me
 ap.add_argument("obscon",
                 help="String matching ONE obscondition in the bitmask yaml file \
                 (e.g. 'BRIGHT'). Controls priorities when merging targets,      \
-                which tiles to process, etc.")
+                which tiles to process, etc.", choices=["DARK", "BRIGHT", "BACKUP"])
 ap.add_argument("-s", "--survey",
                 help="Flavor of survey to run. Defaults to [{}]".format(survey),
-                default=survey)
+                default=survey, choices=["main", "sv3", "sv2"])
 ap.add_argument('--zcatdir',
                 help="Full path to the directory that hosts the redshift        \
                 catalogs. Default is to use the $ZCAT_DIR environment variable",
@@ -33,6 +33,11 @@ ap.add_argument("--reprocess", action='store_true',
 
 ns = ap.parse_args()
 
+if ns.obscon == "BACKUP" and ns.survey == "main":
+    msg = "Updating BACKUP ledgers is not yet supported for the Main Survey!!!"
+    raise RuntimeError(msg)
+    log.error(msg)
+
 scndstates = [False]
 
 # ADM if nosecondary wasn't passed, also process the secondary ledger

doc/changes.rst

@@ -5,9 +5,13 @@ desitarget Change Log
 2.3.1 (unreleased)
 ------------------
 
-* Add optional ``[grz]fiberflux`` arguments to the ``ELG``, ``LRG``, and ``BGS`` 
-  color-cut selection functions to support the `desisim` template-generating
-  code (see `desisim/PR #556`) [`PR #786`].
+* A couple of MTL maintenance updates [`PR #789`_]. Includes:
+    * Turn off BACKUP MTL processing for the Main Survey (for now).
+    * Allow the file format for override ledgers to be forced.
+        * Addresses `issue #784`_.
+* Add optional ``[grz]fiberflux`` arguments to the ``ELG``, ``LRG``, and
+  ``BGS`` color-cut selection functions to support the `desisim`
+  template-generating code (see `desisim/PR #556`_) [`PR #786`_].
 * Extra functionality for splitting randoms by HEALPixel [`PR #785`_]:
     * Correctly process catalogs with an existing ``HPXPIXEL`` column.
 * Functionality to split random catalogs by HEALPixel [`PR #783`_].
@@ -19,7 +23,9 @@ desitarget Change Log
 .. _`PR #783`: https://github.com/desihub/desitarget/pull/783
 .. _`PR #785`: https://github.com/desihub/desitarget/pull/785
 .. _`desisim/PR #556`: https://github.com/desihub/desisim/pull/556
+.. _`issue #784`: https://github.com/desihub/desitarget/issues/784
 .. _`PR #786`: https://github.com/desihub/desitarget/pull/786
+.. _`PR #789`: https://github.com/desihub/desitarget/pull/789
 
 2.3.0 (2021-12-14)
 ------------------

py/desitarget/io.py

@@ -5,6 +5,8 @@
 =============
 
 Functions for reading, writing and manipulating files related to targeting.
+
+.. _`desitarget issue 784`: https://github.com/desihub/desitarget/issues/784#issuecomment-1011452303
 """
 from __future__ import (absolute_import, division)
 #
@@ -3047,24 +3049,27 @@ def read_ecsv_header(filename, cleanup=True):
     return hdr
 
 
-def find_mtl_file_format_from_header(hpdirname, returnoc=False, override=False):
-    """Construct an MTL filename just from the header in the file
+def find_mtl_file_format_from_header(hpdirname, returnoc=False,
+                                     override=False, forceoverride=False):
+    """Construct MTL filename from a directory containing an MTL ledger.
 
     Parameters
     ----------
     hpdirname : :class:`str`
-        Full path to either a directory containing MTL ledgers that have
-        been partitioned by HEALPixel. Or the name of a single ledger.
+        Full path to a directory containing MTL ledgers that have been
+        partitioned by HEALPixel.
     returnoc : :class:`bool`, optional, defaults to ``False``
         If ``True`` then also return the OBSCON header keyword
         for files in this directory.
     override : :class:`bool`, optional, defaults to ``False``
         If ``True``, return the file form for an override ledger instead
-        of a standard MTL ledger IF the location of a standard MTL ledger
-        or ledgers has been passed as `hpdirname`. If the location of an
-        override ledger or ledgers has been passed as `hpdirname`, then
-        the fact that we're working with override ledgers is detected
-        automatically and `override`=``True`` does not need to be passed.
+        of a standard MTL ledger BUT if an OVERRIDE value is detected in
+        the header of the first file found in `hpdirname` let it take
+        precedence over `override`. See `desitarget issue 784`_.
+    forceoverride : :class:`bool`, optional, defaults to ``False``
+        If ``True``, return the file form for an override ledger instead
+        of a standard MTL ledger REGARDLESS of any OVERRIDE value found
+        in the header of the first file found in `hpdirname`.
 
     Returns
     -------
@@ -3078,15 +3083,20 @@ def find_mtl_file_format_from_header(hpdirname, returnoc=False, override=False):
     Notes
     -----
         - Should work for both .ecsv and .fits files.
+        - If both `override` and `forceoverride` are ``True``,
+          `forceoverride` takes precedence.
     """
     # ADM grab information from the target directory.
     surv = read_keyword_from_mtl_header(hpdirname, "SURVEY")
     oc = read_keyword_from_mtl_header(hpdirname, "OBSCON")
     # ADM detect whether we're working with the override ledgers.
-    try:
-        override = read_keyword_from_mtl_header(hpdirname, "OVERRIDE")
-    except KeyError:
-        pass
+    if forceoverride:
+        override = forceoverride
+    else:
+        try:
+            override = read_keyword_from_mtl_header(hpdirname, "OVERRIDE")
+        except KeyError:
+            pass
 
     from desitarget.mtl import get_mtl_ledger_format
     ender = get_mtl_ledger_format()

py/desitarget/mtl.py

@@ -1449,7 +1449,8 @@ def force_overrides(hpdirname, pixlist):
     # ADM find the general format for the ledger files in `hpdirname`.
     fileform = io.find_mtl_file_format_from_header(hpdirname)
     # ADM this is the format for any associated override ledgers.
-    overrideff = io.find_mtl_file_format_from_header(hpdirname, override=True)
+    overrideff = io.find_mtl_file_format_from_header(hpdirname,
+                                                     forceoverride=True)
 
     # ADM before making updates, check all suggested ledgers exist.
     for pix in pixlist:
@@ -1550,7 +1551,8 @@ def reprocess_ledger(hpdirname, zcat, obscon="DARK"):
     # ADM also returning the obsconditions.
     fileform, oc = io.find_mtl_file_format_from_header(hpdirname, returnoc=True)
     # ADM also find the format for any associated override ledgers.
-    overrideff = io.find_mtl_file_format_from_header(hpdirname, override=True)
+    overrideff = io.find_mtl_file_format_from_header(hpdirname,
+                                                     forceoverride=True)
 
     # ADM check the obscondition is as expected.
     if obscon != oc:
@@ -1809,7 +1811,8 @@ def update_ledger(hpdirname, zcat, targets=None, obscon="DARK",
     # ADM also returning the obsconditions.
     fileform, oc = io.find_mtl_file_format_from_header(hpdirname, returnoc=True)
     # ADM this is the format for any associated override ledgers.
-    overrideff = io.find_mtl_file_format_from_header(hpdirname, override=True)
+    overrideff = io.find_mtl_file_format_from_header(hpdirname,
+                                                     forceoverride=True)
 
     # ADM check the obscondition is as expected.
     if obscon != oc: