Merge branch 'main' into dark_model

desihub · Aug 17, 2024 · 3def59d · 3def59d
2 parents 7c7d02d + 2f63761
commit 3def59d
Show file tree

Hide file tree

Showing 33 changed files with 1,623 additions and 423 deletions.
diff --git a/bin/desi_job_graph b/bin/desi_job_graph
@@ -83,6 +83,7 @@ fx.write(f"""<!DOCTYPE html>
     classDef OUT_OF_MEMORY fill:#d95f0e;
     classDef TIMEOUT       fill:#d95f0e;
     classDef CANCELLED     fill:#fed98e;
+    classDef NOTSUBMITTED  fill:#fcae1e;
     classDef UNKNOWN       fill:#ffffcc;
 """)
 
@@ -106,6 +107,8 @@ for row in proctable:
 
     if qid in jobinfo:
         state = jobinfo[qid]['STATE'].split()[0]
+    elif qid == 1:
+        state = 'NOTSUBMITTED'
     else:
         state = 'UNKNOWN'
 

diff --git a/bin/desi_merge_psf b/bin/desi_merge_psf
@@ -12,6 +12,4 @@ parser.add_argument('-o','--outfile', type = str, default = None, required=True,
 
 args = parser.parse_args()
 
-merge_psf(args.infiles,args.outfile)
-
-
+merge_psf(args.infiles[0],args.infiles[1:],args.outfile)
diff --git a/bin/desi_resubmit_queue_failures b/bin/desi_resubmit_queue_failures
@@ -11,9 +11,10 @@ from astropy.table import Table
 import glob
 
 ## Import some helper functions, you can see their definitions by uncomenting the bash shell command
+from desiutil.log import get_logger
 from desispec.workflow.tableio import load_table, write_table
 from desispec.workflow.proctable import get_processing_table_pathname
-from desispec.workflow.processing import update_and_recurvsively_submit
+from desispec.workflow.processing import update_and_recursively_submit
 from desispec.workflow.queue import get_resubmission_states
 
 def parse_args():  # options=None):
@@ -49,6 +50,7 @@ def parse_args():  # options=None):
 
 if __name__ == '__main__':
     args = parse_args()
+    log = get_logger()
     ptable_pathname = args.proc_table_pathname
     if ptable_pathname is None:
         if args.night is None:
@@ -66,21 +68,25 @@ if __name__ == '__main__':
         if not args.dont_resub_failed:
             resub_states.append('FAILED')
 
-    print(f"Resubmitting the following Slurm states: {resub_states}")
+    log.info(f"Resubmitting the following Slurm states: {resub_states}")
+
+    if args.dry_run > 0 and args.dry_run < 3:
+        log.warning(f"{args.dry_run=} will be run with limited simulation "
+                    f"because we don't want to write out incorrect queue information.")
 
     ## Combine the table names and types for easier passing to io functions
     table_type = 'proctable'
 
     ## Load in the files defined above
     ptable = load_table(tablename=ptable_pathname, tabletype=table_type)
-    print(f"Identified ptable with {len(ptable)} entries.")
-    ptable, nsubmits = update_and_recurvsively_submit(ptable, submits=0,
-                                   resubmission_states=resub_states,
-                                   ptab_name=ptable_pathname, dry_run=args.dry_run,
-                                   reservation=args.reservation)
+    log.info(f"Identified ptable with {len(ptable)} entries.")
+    ptable, nsubmits = update_and_recursively_submit(ptable, submits=0,
+                                                     resubmission_states=resub_states,
+                                                     ptab_name=ptable_pathname, dry_run=args.dry_run,
+                                                     reservation=args.reservation)
 
     if not args.dry_run:
         write_table(ptable, tablename=ptable_pathname)
 
-    print("Completed all necessary queue resubmissions from processing "
+    log.info("Completed all necessary queue resubmissions from processing "
           + f"table: {ptable_pathname}")
diff --git a/bin/desi_run_prod b/bin/desi_run_prod
diff --git a/bin/desi_submit_prod b/bin/desi_submit_prod
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import argparse
+
+import sys
+from desispec.scripts.submit_prod import submit_production
+
+def parse_args():  # options=None):
+    """
+    Creates an arguments parser for the desi run production
+    """
+    parser = argparse.ArgumentParser(description="Submit a full production run of the DESI data pipeline for processing.")
+
+    parser.add_argument("-p", "--production-yaml", type=str, required=True,
+                        help="Relative or absolute pathname to the yaml file summarizing the production.")
+    parser.add_argument("-q", "--queue-threshold", type=int, default=4500,
+                        help="The number of jobs for the current user in the queue at which the"
+                             + " at which the script stops submitting new jobs.")
+
+    # Code Flags
+    parser.add_argument("--dry-run-level", type=int, default=0,
+                        help="Perform a dry run where no jobs are actually created or submitted."
+                             + " Give what --dry-run-level to pass to desi_proc_night.")
+    # parser.add_argument("--error-if-not-available", action="store_true",
+    #                     help="Raise an error instead of reporting and moving on if an exposure "+\
+    #                          "table doesn't exist.")
+
+    args = parser.parse_args()
+
+    return args
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    sys.exit(submit_production(**args.__dict__))
+
diff --git a/doc/changes.rst b/doc/changes.rst
@@ -2,7 +2,12 @@
 desispec Change Log
 ===================
 
-0.64.1 (unreleased)
+0.65.1 (unreleased)
+-------------------
+
+* No changes yet.
+
+0.65.0 (2024-08-16)
 -------------------
 
 * healpix grouping memory and I/O efficiency improvements (PR `#2290`_).
@@ -21,6 +26,13 @@ desispec Change Log
   of DESI_SPECTRO_CALIB yaml files (PR `#2313`_).
 * Use read noise estimated per CCD row for some amplifiers (PR `#2314`_).
 * Allow certain old redrock files in daily reductions to be read (PR `#2317`_).
+* Fix assemble_fibermap and unit tests for old data without dark models
+  (PR `#2319`_).
+* merge_psf option to not include input psf for sims (PR `#2320`_).
+* Add cross-night dependency tracking for cumulative ztile jobs (PR `#2321`_).
+* Modernize submit_prod (PR `#2322`_).
+* FIBERSTATUS VARIABLETHRU bad for sky and stdstars (PR `#2323`_).
+* faflavor2program for special tiles: fix other to bright or dark (PR `#2325`_).
 
 .. _`#2290`: https://github.com/desihub/desispec/pull/2290
 .. _`#2294`: https://github.com/desihub/desispec/pull/2294
@@ -31,6 +43,12 @@ desispec Change Log
 .. _`#2313`: https://github.com/desihub/desispec/pull/2313
 .. _`#2314`: https://github.com/desihub/desispec/pull/2314
 .. _`#2317`: https://github.com/desihub/desispec/pull/2317
+.. _`#2319`: https://github.com/desihub/desispec/pull/2319
+.. _`#2320`: https://github.com/desihub/desispec/pull/2320
+.. _`#2321`: https://github.com/desihub/desispec/pull/2321
+.. _`#2322`: https://github.com/desihub/desispec/pull/2322
+.. _`#2323`: https://github.com/desihub/desispec/pull/2323
+.. _`#2325`: https://github.com/desihub/desispec/pull/2325
 
 0.64.0 (2024-07-01)
 -------------------

diff --git a/py/desispec/_version.py b/py/desispec/_version.py
@@ -1 +1 @@
-__version__ = '0.64.0.dev8624'
+__version__ = '0.65.0.dev8717'
diff --git a/py/desispec/fiberbitmasking.py b/py/desispec/fiberbitmasking.py
@@ -110,7 +110,7 @@ def get_fiberbitmasked_frame_arrays(frame,bitmask=None,ivar_framemask=True,retur
 
 def get_fiberbitmask_comparison_value(kind,band):
     """Takes a string argument and returns a 32-bit integer representing the logical OR of all
-    relevant fibermask bits for that given reduction step
+    fatally bad fibermask bits for that given reduction step
 
     Args:
         kind: str : string designating which combination of bits to use based on the operation.
@@ -122,6 +122,8 @@ def get_fiberbitmask_comparison_value(kind,band):
     Returns:
         bitmask : 32 bit bitmask corresponding to the fiberbitmask of the desired kind
             in the desired cameras (bands).
+
+    if FIBERSTATUS & bitmask != 0, then that fiber should not be used
     """
     if kind.lower() == 'all':
         return get_all_fiberbitmask_with_amp(band)
@@ -141,17 +143,33 @@ def get_fiberbitmask_comparison_value(kind,band):
 
 
 def get_skysub_fiberbitmask_val(band):
-    return get_all_fiberbitmask_with_amp(band)
+    """
+    Return mask of bad FIBERSTATUS bits for selecting sky fibers,
+    i.e. fibers with these bits set should not be used for the sky model
+    """
+    return get_all_fiberbitmask_with_amp(band) | fmsk.VARIABLETHRU
 
 def get_flat_fiberbitmask_val(band):
+    """
+    Return mask of bad FIBERSTATUS bits for fiberflats
+    i.e. fibers with these bits set have a bad fiberflat and cannot be used
+    """
     return (fmsk.BROKENFIBER | fmsk.BADFIBER | fmsk.BADTRACE | fmsk.BADARC | \
             fmsk.MANYBADCOL | fmsk.MANYREJECTED )
 
 def get_fluxcalib_fiberbitmask_val(band):
+    """
+    Return mask of bad FIBERSTATUS bits that should trigger flux=ivar=0
+    instead of flux calibrating the spectra.
+    """
     return get_all_fiberbitmask_with_amp(band)
 
 def get_stdstars_fiberbitmask_val(band):
-    return get_all_fiberbitmask_with_amp(band) | fmsk.POORPOSITION
+    """
+    Return mask of bad FIBERSTATUS bits for selecting standard stars,
+    i.e. fibers with these bits set should not be used as standard stars
+    """
+    return get_all_fiberbitmask_with_amp(band) | fmsk.POORPOSITION | fmsk.VARIABLETHRU
 
 def get_all_nonamp_fiberbitmask_val():
     """Return a mask for all fatally bad FIBERSTATUS bits except BADAMPB/R/Z
@@ -161,16 +179,25 @@ def get_all_nonamp_fiberbitmask_val():
         be on a valid sky location, or even a target for RESTRICTED.
         Also does not include POORPOSITION which is bad for stdstars
         but not necessarily fatal for otherwise processing a normal fiber.
+        NEARCHARGETRAP and VARIABLETHRU are also not included since
+        they are ok for some types of processing but not others.
     """
     return (fmsk.BROKENFIBER | fmsk.MISSINGPOSITION | \
             fmsk.BADPOSITION | \
             fmsk.BADFIBER | fmsk.BADTRACE | fmsk.BADARC | fmsk.BADFLAT | \
             fmsk.MANYBADCOL | fmsk.MANYREJECTED )
 
 def get_justamps_fiberbitmask():
+    """
+    Return a mask of the amp-specific FIBERSTATUS bits
+    """
     return ( fmsk.BADAMPB | fmsk.BADAMPR | fmsk.BADAMPZ )
 
 def get_all_fiberbitmask_with_amp(band):
+    """
+    Return all fatally bad FIBERSTATUS bits including the amp-specific
+    bit for this band
+    """
     amp_mask = get_all_nonamp_fiberbitmask_val()
     if band.lower().find('b')>=0:
         amp_mask |= fmsk.BADAMPB
@@ -181,4 +208,7 @@ def get_all_fiberbitmask_with_amp(band):
     return amp_mask
 
 def get_all_fiberbitmask_val():
+    """
+    Return a mask of all fatally bad FIBERSTATUS bits
+    """
     return ( get_all_nonamp_fiberbitmask_val() | get_justamps_fiberbitmask() )
diff --git a/py/desispec/io/meta.py b/py/desispec/io/meta.py
@@ -790,7 +790,10 @@ def faflavor2program(faflavor):
         faprgm (str or array of str): what FAPRGM would be if we had set it
         (dark, bright, backup, other)
 
-    Note: this was standardized by sv3 and main, but evolved during sv1 and sv2
+    Note: this was standardized by sv3 and main, but evolved during sv1 and sv2.
+        for the survey=special tiles (m31, odin, and tertiary), the info
+        is/can be retrieve from the GOALTYPE keyword in the zero-th extension
+        of the fiberassign file.
     """
     #- Handle scalar or array input, upcasting bytes to str as needed
     scalar_input = np.isscalar(faflavor)
@@ -807,11 +810,36 @@ def faflavor2program(faflavor):
     dark |= faflavor == 'sv1elgqso'
     dark |= faflavor == 'sv1lrgqso'
     dark |= faflavor == 'sv1lrgqso2'
+    dark |= np.in1d(
+        faflavor,
+        np.char.add(
+            "special",
+            [
+                'm31', 'odin', 'tertiary1', 'tertiary2', 'tertiary4', 'tertiary5',
+                'tertiary7', 'tertiary9', 'tertiary11', 'tertiary14', 'tertiary15',
+                'tertiary16', 'tertiary17', 'tertiary18', 'tertiary21', 'tertiary23',
+                'tertiary25', 'tertiary26', 'tertiary27', 'tertiary31', 'tertiary35',
+                'tertiary37', 'tertiary38', 'tertiary40', 'tertiary41',
+            ]
+        )
+    )
     dark |= np.char.endswith(faflavor, 'dark')
 
     #- SV1 FAFLAVOR options that map to FAPRGRM='bright'
     bright  = faflavor == 'sv1bgsmws'
     bright |= (faflavor != 'sv1unwisebluebright') & np.char.endswith(faflavor, 'bright')
+    bright |= np.in1d(
+        faflavor,
+        np.char.add(
+            "special",
+            [
+                'tertiary3', 'tertiary6', 'tertiary8', 'tertiary10', 'tertiary12',
+                'tertiary13', 'tertiary19', 'tertiary20', 'tertiary22', 'tertiary24',
+                'tertiary28', 'tertiary29', 'tertiary30', 'tertiary32', 'tertiary33',
+                'tertiary34', 'tertiary36', 'tertiary39',
+            ]
+        )
+    )
 
     #- SV1 FAFLAVOR options that map to FAPRGRM='backup'
     backup  = faflavor == 'sv1backup1'

diff --git a/py/desispec/scripts/daily_processing.py b/py/desispec/scripts/daily_processing.py
@@ -23,7 +23,7 @@
                                         erow_to_prow, default_prow
 from desispec.workflow.processing import parse_previous_tables, flat_joint_fit, arc_joint_fit, get_type_and_tile, \
                                         science_joint_fit, define_and_assign_dependency, create_and_submit, \
-                                        update_and_recurvsively_submit, checkfor_and_submit_joint_job, \
+                                        update_and_recursively_submit, checkfor_and_submit_joint_job, \
                                         submit_tilenight_and_redshifts
 from desispec.workflow.queue import update_from_queue, any_jobs_not_complete
 from desispec.io.util import difference_camwords, parse_badamps, validate_badamps
@@ -445,7 +445,7 @@ def daily_processing_manager(specprod=None, exp_table_path=None, proc_table_path
 
         if len(ptable) > 0:
             ptable = update_from_queue(ptable, dry_run=dry_run_level)
-            # ptable, nsubmits = update_and_recurvsively_submit(ptable,
+            # ptable, nsubmits = update_and_recursively_submit(ptable,
             #                                                   ptab_name=proc_table_pathname, dry_run=dry_run_level)
 
             ## Exposure table doesn't change in the interim, so no need to re-write it to disk
@@ -500,7 +500,7 @@ def daily_processing_manager(specprod=None, exp_table_path=None, proc_table_path
     # ii,nsubmits = 0, 0
     # while ii < 4 and any_jobs_not_complete(ptable['STATUS']):
     #     print(f"Starting iteration {ii} of queue updating and resubmissions of failures.")
-    #     ptable, nsubmits = update_and_recurvsively_submit(ptable, submits=nsubmits,
+    #     ptable, nsubmits = update_and_recursively_submit(ptable, submits=nsubmits,
     #                                                       ptab_name=proc_table_pathname, dry_run=dry_run_level)
     #     if dry_run_level < 3:
     #          write_table(ptable, tablename=proc_table_pathname)

diff --git a/py/desispec/scripts/proc.py b/py/desispec/scripts/proc.py
@@ -645,6 +645,9 @@ def main(args=None, comm=None):
                 cmd += ' --input-psf {}'.format(inpsf)
                 cmd += ' --output-psf {}'.format(outpsf)
 
+                if args.dont_merge_with_psf_input :
+                    cmd += ' --dont-merge-with-input'
+
                 # fibers to ignore for the PSF fit
                 # specex uses the fiber index in a camera
                 fibers_to_ignore = badfibers([hdr, camhdr[camera]],["BROKENFIBERS","BADCOLUMNFIBERS"])%500