From 7b215e95b3635b3185bc07489e1d70617eccd7ca Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 13 Jan 2022 18:52:30 -0800 Subject: [PATCH 1/4] add forceoverride kwarg to find_mtl_file_format_from_header() to indicate when we REALLY want the override file format --- py/desitarget/io.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/py/desitarget/io.py b/py/desitarget/io.py index c1a6e8beb..b88cf0cfa 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -5,6 +5,8 @@ ============= Functions for reading, writing and manipulating files related to targeting. + +.. _`desitarget issue 784`: https://github.com/desihub/desitarget/issues/784#issuecomment-1011452303 """ from __future__ import (absolute_import, division) # @@ -3047,24 +3049,27 @@ def read_ecsv_header(filename, cleanup=True): return hdr -def find_mtl_file_format_from_header(hpdirname, returnoc=False, override=False): - """Construct an MTL filename just from the header in the file +def find_mtl_file_format_from_header(hpdirname, returnoc=False, + override=False, forceoverride=False): + """Construct MTL filename from a directory containing an MTL ledger. Parameters ---------- hpdirname : :class:`str` - Full path to either a directory containing MTL ledgers that have - been partitioned by HEALPixel. Or the name of a single ledger. + Full path to a directory containing MTL ledgers that have been + partitioned by HEALPixel. returnoc : :class:`bool`, optional, defaults to ``False`` If ``True`` then also return the OBSCON header keyword for files in this directory. override : :class:`bool`, optional, defaults to ``False`` If ``True``, return the file form for an override ledger instead - of a standard MTL ledger IF the location of a standard MTL ledger - or ledgers has been passed as `hpdirname`. If the location of an - override ledger or ledgers has been passed as `hpdirname`, then - the fact that we're working with override ledgers is detected - automatically and `override`=``True`` does not need to be passed. + of a standard MTL ledger BUT if an OVERRIDE value is detected in + the header of the first file found in `hpdirname` let it take + precedence over `override`. See `desitarget issue 784`_. + forceoverride : :class:`bool`, optional, defaults to ``False`` + If ``True``, return the file form for an override ledger instead + of a standard MTL ledger REGARDLESS of any OVERRIDE value found + in the header of the first file found in `hpdirname`. Returns ------- @@ -3078,15 +3083,20 @@ def find_mtl_file_format_from_header(hpdirname, returnoc=False, override=False): Notes ----- - Should work for both .ecsv and .fits files. + - If both `override` and `forceoverride` are ``True``, + `forceoverride` takes precedence. """ # ADM grab information from the target directory. surv = read_keyword_from_mtl_header(hpdirname, "SURVEY") oc = read_keyword_from_mtl_header(hpdirname, "OBSCON") # ADM detect whether we're working with the override ledgers. - try: - override = read_keyword_from_mtl_header(hpdirname, "OVERRIDE") - except KeyError: - pass + if forceoverride: + override = forceoverride + else: + try: + override = read_keyword_from_mtl_header(hpdirname, "OVERRIDE") + except KeyError: + pass from desitarget.mtl import get_mtl_ledger_format ender = get_mtl_ledger_format() From 05c6b4ba0ef92a9ae074d278118cce1b4ad401d4 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 14 Jan 2022 09:41:02 -0800 Subject: [PATCH 2/4] always force the override ledger format when looking for that format in MTL --- py/desitarget/mtl.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index 76c121291..ff9a9e91b 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -1449,7 +1449,8 @@ def force_overrides(hpdirname, pixlist): # ADM find the general format for the ledger files in `hpdirname`. fileform = io.find_mtl_file_format_from_header(hpdirname) # ADM this is the format for any associated override ledgers. - overrideff = io.find_mtl_file_format_from_header(hpdirname, override=True) + overrideff = io.find_mtl_file_format_from_header(hpdirname, + forceoverride=True) # ADM before making updates, check all suggested ledgers exist. for pix in pixlist: @@ -1550,7 +1551,8 @@ def reprocess_ledger(hpdirname, zcat, obscon="DARK"): # ADM also returning the obsconditions. fileform, oc = io.find_mtl_file_format_from_header(hpdirname, returnoc=True) # ADM also find the format for any associated override ledgers. - overrideff = io.find_mtl_file_format_from_header(hpdirname, override=True) + overrideff = io.find_mtl_file_format_from_header(hpdirname, + forceoverride=True) # ADM check the obscondition is as expected. if obscon != oc: @@ -1809,7 +1811,8 @@ def update_ledger(hpdirname, zcat, targets=None, obscon="DARK", # ADM also returning the obsconditions. fileform, oc = io.find_mtl_file_format_from_header(hpdirname, returnoc=True) # ADM this is the format for any associated override ledgers. - overrideff = io.find_mtl_file_format_from_header(hpdirname, override=True) + overrideff = io.find_mtl_file_format_from_header(hpdirname, + forceoverride=True) # ADM check the obscondition is as expected. if obscon != oc: From 873faff0dd3853d79222e93d84f1cd105e50079e Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 14 Jan 2022 15:37:48 -0800 Subject: [PATCH 3/4] temporarily turn off MTL for BACKUP tiles for the Main Survey --- bin/run_mtl_loop | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/run_mtl_loop b/bin/run_mtl_loop index 7bc9a9bff..7bd5cc5eb 100755 --- a/bin/run_mtl_loop +++ b/bin/run_mtl_loop @@ -12,10 +12,10 @@ ap = ArgumentParser(description='Make an initial HEALPixel-split ledger for a Me ap.add_argument("obscon", help="String matching ONE obscondition in the bitmask yaml file \ (e.g. 'BRIGHT'). Controls priorities when merging targets, \ - which tiles to process, etc.") + which tiles to process, etc.", choices=["DARK", "BRIGHT", "BACKUP"]) ap.add_argument("-s", "--survey", help="Flavor of survey to run. Defaults to [{}]".format(survey), - default=survey) + default=survey, choices=["main", "sv3", "sv2"]) ap.add_argument('--zcatdir', help="Full path to the directory that hosts the redshift \ catalogs. Default is to use the $ZCAT_DIR environment variable", @@ -33,6 +33,11 @@ ap.add_argument("--reprocess", action='store_true', ns = ap.parse_args() +if ns.obscon == "BACKUP" and ns.survey == "main": + msg = "Updating BACKUP ledgers is not yet supported for the Main Survey!!!" + raise RuntimeError(msg) + log.error(msg) + scndstates = [False] # ADM if nosecondary wasn't passed, also process the secondary ledger From ad5cfc0a86de0a91038751eadae5b656ebadaaae Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 14 Jan 2022 16:16:36 -0800 Subject: [PATCH 4/4] update the changes docs --- doc/changes.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/changes.rst b/doc/changes.rst index e9d3f5c41..4d95d0168 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,9 +5,13 @@ desitarget Change Log 2.3.1 (unreleased) ------------------ -* Add optional ``[grz]fiberflux`` arguments to the ``ELG``, ``LRG``, and ``BGS`` - color-cut selection functions to support the `desisim` template-generating - code (see `desisim/PR #556`) [`PR #786`]. +* A couple of MTL maintenance updates [`PR #789`_]. Includes: + * Turn off BACKUP MTL processing for the Main Survey (for now). + * Allow the file format for override ledgers to be forced. + * Addresses `issue #784`_. +* Add optional ``[grz]fiberflux`` arguments to the ``ELG``, ``LRG``, and + ``BGS`` color-cut selection functions to support the `desisim` + template-generating code (see `desisim/PR #556`_) [`PR #786`_]. * Extra functionality for splitting randoms by HEALPixel [`PR #785`_]: * Correctly process catalogs with an existing ``HPXPIXEL`` column. * Functionality to split random catalogs by HEALPixel [`PR #783`_]. @@ -19,7 +23,9 @@ desitarget Change Log .. _`PR #783`: https://github.com/desihub/desitarget/pull/783 .. _`PR #785`: https://github.com/desihub/desitarget/pull/785 .. _`desisim/PR #556`: https://github.com/desihub/desisim/pull/556 +.. _`issue #784`: https://github.com/desihub/desitarget/issues/784 .. _`PR #786`: https://github.com/desihub/desitarget/pull/786 +.. _`PR #789`: https://github.com/desihub/desitarget/pull/789 2.3.0 (2021-12-14) ------------------