Merge pull request #2313 from desihub/more-fiberstatus-bits

More FIBERSTATUS bits

doc/changes.rst

@@ -16,13 +16,18 @@ desispec Change Log
 * zproc requires exposure-qa files for tileqa step (PR `#2306`_).
 * Don't set envs in desispec.module that are now set in desimodules
   (PR `#2310`_).
+* New FIBERSTATUS NEARCHARGETRAP and VARIABLETHRU set in
+  desispec.io.fibermap.assemble_fibermap based on content
+  of DESI_SPECTRO_CALIB yaml files (PR `#2313`_).
 
 .. _`#2290`: https://github.com/desihub/desispec/pull/2290
 .. _`#2294`: https://github.com/desihub/desispec/pull/2294
 .. _`#2296`: https://github.com/desihub/desispec/pull/2296
 .. _`#2302`: https://github.com/desihub/desispec/pull/2302
 .. _`#2306`: https://github.com/desihub/desispec/pull/2306
 .. _`#2310`: https://github.com/desihub/desispec/pull/2310
+.. _`#2313`: https://github.com/desihub/desispec/pull/2313
+
 
 0.64.0 (2024-07-01)
 -------------------

py/desispec/calibfinder.py

@@ -148,16 +148,17 @@ def ccdregionmask(headers) :
         masks.append(mask)
     return masks
 
+badfiber_keywords=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS","BADAMPFIBERS","EXCLUDEFIBERS","NEARCHARGETRAPFIBERS", "VARIABLETHRUFIBERS"]
 
-def badfibers(headers,keys=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS"],yaml_file=None) :
+def badfibers(headers,keys=badfiber_keywords,yaml_file=None) :
     """
     find list of bad fibers from $DESI_SPECTRO_CALIB using the keywords found in the headers
 
     Args:
         headers: list of fits headers, or list of dictionnaries
 
     Optional:
-        keys: list of keywords, among ["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS"]. Default is all of them.
+        keys: list of keywords, among calibfinder.badfiber_keywords. Default is all of them.
         yaml_file: path to a specific yaml file. By default, the code will
         automatically find the yaml file from the environment variable
         DESI_SPECTRO_CALIB and the CAMERA keyword in the headers
@@ -386,17 +387,16 @@ def findfile(self,key) :
         """
         return os.path.join(self.directory,self.data[key])
 
-    def badfibers(self,keys=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS","BADAMPFIBERS","EXCLUDEFIBERS"]) :
+    def badfibers(self,keys=badfiber_keywords) :
         """
         Args:
-            keys: optional, list of keywords, among BROKENFIBERS,BADCOLUMNFIBERS,LOWTRANSMISSIONFIBERS,BADAMPFIBERS,EXCLUDEFIBERS. Default is all of them.
+            keys: optional, list of keywords, among calibfinder.badfiber_keywords. Default is all of them.
 
         Returns:
             List of bad fibers from yaml file as a 1D array of intergers
         """
         log = get_logger()
         fibers=[]
-        badfiber_keywords=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS","BADAMPFIBERS","EXCLUDEFIBERS"]
         for key in keys :
             if key not in badfiber_keywords  :
                 log.error(f"key '{key}' not in the list of valid keys for bad fibers: {validkeys}")
@@ -512,7 +512,7 @@ def find_darks_in_desi_spectro_dark(self, header):
                             continue
                         else:
                             log.debug(f'Temperature difference to selected dark is {np.abs(dark_entry["CCDTEMP"] - header["CCDTEMP"]):.5f}')
-                    
+
                     #same for bias
                     if bias_entry["DETECTOR"].strip() != self.data["DETECTOR"].strip() :
                         log.debug("Skip file %s with DETECTOR=%s != %s"%(bias_entry["FILENAME"],bias_entry["DETECTOR"],self.data["DETECTOR"]))
@@ -531,7 +531,7 @@ def find_darks_in_desi_spectro_dark(self, header):
                             continue
                         else:
                             log.debug(f'Temperature difference to selected bias is {np.abs(bias_entry["CCDTEMP"] - header["CCDTEMP"]):.5f}')
-                    
+
                     found=True
                     log.debug(f"Found matching dark frames for camera {cameraid} created on {date_used}")
                     break
@@ -562,4 +562,3 @@ def find_darks_in_desi_spectro_dark(self, header):
                 #this would prevent nightwatch failures in case of not-yet-existing files
                 log.error(f"Didn't find matching {camera} calibration darks in $DESI_SPECTRO_DARK, "
                            "falling back to $DESI_SPECTRO_CALIB")
-

py/desispec/io/fibermap.py

@@ -26,6 +26,7 @@
 from desispec.io.util import (fitsheader, write_bintable, makepath, addkeys,
     parse_badamps, checkgzip)
 from desispec.io.meta import rawdata_root, findfile
+from desispec.calibfinder import CalibFinder
 
 from . import iotime
 from .table import read_table
@@ -1103,6 +1104,52 @@ def assemble_fibermap(night, expid, badamps=None, badfibers_filename=None,
             badfibermask = np.bitwise_or.reduce(table["FIBERSTATUS"][table["FIBER"]==fiber])
             fibermap['FIBERSTATUS'][fibermap["FIBER"]==fiber] |= badfibermask
 
+    #- Add FIBERSTATUS bits based on DESI_SPECTRO_CALIB entries for all cameras
+    badfibers_keywords_and_maskbits={"NEARCHARGETRAPFIBERS":fibermask.NEARCHARGETRAP,"VARIABLETHRUFIBERS":fibermask.VARIABLETHRU}
+    badfibers=dict()
+    for key in badfibers_keywords_and_maskbits.keys() :
+        badfibers[key]=np.empty(0,dtype=int)
+    #- loop over cameras to look for bad fibers in the calibration yaml files with a calibfinder
+    for specid in range(10) :
+        for arm in ["b","r","z"] :
+            camera="{}{}".format(arm,specid)
+            try:
+                camheader = fits.getheader(rawfile, camera, disable_image_compression=True)
+            except KeyError:
+                log.debug("No camera {} in this file".format(camera))
+                continue
+            cfinder=CalibFinder([rawheader,camheader])
+            for key in badfibers_keywords_and_maskbits.keys() :
+                newbadfibers = cfinder.badfibers([key])
+                if newbadfibers.size > 0 :
+                    log.debug("Adding {} bad fibers {} of camera {}".format(key,newbadfibers,camera))
+                    # check that they do match the fiber number of this spectrograph or throw error
+                    # to avoid any misunderstanding
+                    if np.any((newbadfibers<specid*500)|(newbadfibers>=(specid+1)*500)) :
+                        mess="Not all fibers from {} belong to spectrograph ID {}".format(newbadfibers,specid)
+                        log.error(mess)
+                        raise KeyError(mess)
+                    badfibers[key] = np.append(badfibers[key],newbadfibers)
+
+    #- remove possible duplicates
+    # (it can happen if a bad fiber is mentionned in several cameras of the same spectro)
+    for key in badfibers_keywords_and_maskbits.keys() :
+        if badfibers[key].size > 1 :
+            badfibers[key] = np.unique(badfibers[key])
+    #- nice message
+    message="Bad fibers from DESI_SPECTRO_CALIB:"
+    is_empty=True
+    for key in badfibers_keywords_and_maskbits.keys() :
+        if badfibers[key].size > 0 :
+            message += " {}:{}".format(key,badfibers[key])
+            is_empty=False
+    if is_empty : message += " None"
+    log.info(message)
+    #- now add the keywords to FIBERSTATUS
+    for key in badfibers_keywords_and_maskbits.keys() :
+        selection = np.in1d(fibermap["FIBER"],badfibers[key])
+        fibermap["FIBERSTATUS"][selection] |= badfibers_keywords_and_maskbits[key]
+
     #- NaN are a pain; reset to dummy values
     for col in [
         'FIBER_X', 'FIBER_Y',

py/desispec/maskbits.py

@@ -62,6 +62,8 @@
     - [BADPOSITION,     9, "Fiber >100 microns from target location"]
     - [POORPOSITION,   10, "Fiber >30 microns from target location"]
     - [LOWTRANSMISSION, 12, "Low fiber transmission. Cannot use for sky."]
+    - [NEARCHARGETRAP, 13, "Fiber trace near charge trap in one of the CCDs"]
+    - [VARIABLETHRU,   14, "Fiber has throughput variations we cannot model well"]
     - [LOWEFFTIME,     15, "Effective time for this fiber is too low"]
     - [BADFIBER,       16, "Unusable fiber"]
     - [BADTRACE,       17, "Bad trace solution"]