From edc6f41f010312c387af4561d367b568f9286f07 Mon Sep 17 00:00:00 2001 From: Julien Guy Date: Thu, 8 Aug 2024 09:30:18 -0700 Subject: [PATCH 1/5] adding 2 fibermask bits: NEARCHARGETRAP and VARIABLETHRU --- py/desispec/maskbits.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/py/desispec/maskbits.py b/py/desispec/maskbits.py index b27cec5a4..634b3639b 100644 --- a/py/desispec/maskbits.py +++ b/py/desispec/maskbits.py @@ -62,6 +62,8 @@ - [BADPOSITION, 9, "Fiber >100 microns from target location"] - [POORPOSITION, 10, "Fiber >30 microns from target location"] - [LOWTRANSMISSION, 12, "Low fiber transmission. Cannot use for sky."] + - [NEARCHARGETRAP, 13, "Fiber trace near charge trap in one of the CCDs"] + - [VARIABLETHRU, 14, "Fiber has throughput variations we cannot model well"] - [LOWEFFTIME, 15, "Effective time for this fiber is too low"] - [BADFIBER, 16, "Unusable fiber"] - [BADTRACE, 17, "Bad trace solution"] From 920cb2b7d26918510d50f14929b7da5047a4c502 Mon Sep 17 00:00:00 2001 From: Julien Guy Date: Thu, 8 Aug 2024 09:48:31 -0700 Subject: [PATCH 2/5] add "NEARCHARGETRAPFIBERS","VARIABLETHRUFIBERS" to the list of possible keys for routine badfibers(keys) --- py/desispec/calibfinder.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/py/desispec/calibfinder.py b/py/desispec/calibfinder.py index 07db09c41..5660be044 100644 --- a/py/desispec/calibfinder.py +++ b/py/desispec/calibfinder.py @@ -386,7 +386,7 @@ def findfile(self,key) : """ return os.path.join(self.directory,self.data[key]) - def badfibers(self,keys=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS","BADAMPFIBERS","EXCLUDEFIBERS"]) : + def badfibers(self,keys=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS","BADAMPFIBERS","EXCLUDEFIBERS","NEARCHARGETRAPFIBERS", "VARIABLETHRUFIBERS"]) : """ Args: keys: optional, list of keywords, among BROKENFIBERS,BADCOLUMNFIBERS,LOWTRANSMISSIONFIBERS,BADAMPFIBERS,EXCLUDEFIBERS. Default is all of them. @@ -396,7 +396,7 @@ def badfibers(self,keys=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS """ log = get_logger() fibers=[] - badfiber_keywords=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS","BADAMPFIBERS","EXCLUDEFIBERS"] + badfiber_keywords=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS","BADAMPFIBERS","EXCLUDEFIBERS","NEARCHARGETRAPFIBERS","VARIABLETHRUFIBERS"] for key in keys : if key not in badfiber_keywords : log.error(f"key '{key}' not in the list of valid keys for bad fibers: {validkeys}") @@ -512,7 +512,7 @@ def find_darks_in_desi_spectro_dark(self, header): continue else: log.debug(f'Temperature difference to selected dark is {np.abs(dark_entry["CCDTEMP"] - header["CCDTEMP"]):.5f}') - + #same for bias if bias_entry["DETECTOR"].strip() != self.data["DETECTOR"].strip() : log.debug("Skip file %s with DETECTOR=%s != %s"%(bias_entry["FILENAME"],bias_entry["DETECTOR"],self.data["DETECTOR"])) @@ -531,7 +531,7 @@ def find_darks_in_desi_spectro_dark(self, header): continue else: log.debug(f'Temperature difference to selected bias is {np.abs(bias_entry["CCDTEMP"] - header["CCDTEMP"]):.5f}') - + found=True log.debug(f"Found matching dark frames for camera {cameraid} created on {date_used}") break @@ -562,4 +562,3 @@ def find_darks_in_desi_spectro_dark(self, header): #this would prevent nightwatch failures in case of not-yet-existing files log.error(f"Didn't find matching {camera} calibration darks in $DESI_SPECTRO_DARK, " "falling back to $DESI_SPECTRO_CALIB") - From 506b36104fa840ca373818cf32f2808450041158 Mon Sep 17 00:00:00 2001 From: Julien Guy Date: Thu, 8 Aug 2024 10:53:40 -0700 Subject: [PATCH 3/5] define calibfinder.badfiber_keywords and use it in calibfiner.badfibers() and calibfinder.CalibFinder.badfibers() --- py/desispec/calibfinder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/py/desispec/calibfinder.py b/py/desispec/calibfinder.py index 5660be044..b1a33b7ee 100644 --- a/py/desispec/calibfinder.py +++ b/py/desispec/calibfinder.py @@ -148,8 +148,9 @@ def ccdregionmask(headers) : masks.append(mask) return masks +badfiber_keywords=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS","BADAMPFIBERS","EXCLUDEFIBERS","NEARCHARGETRAPFIBERS", "VARIABLETHRUFIBERS"] -def badfibers(headers,keys=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS"],yaml_file=None) : +def badfibers(headers,keys=badfiber_keywords,yaml_file=None) : """ find list of bad fibers from $DESI_SPECTRO_CALIB using the keywords found in the headers @@ -157,7 +158,7 @@ def badfibers(headers,keys=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIB headers: list of fits headers, or list of dictionnaries Optional: - keys: list of keywords, among ["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS"]. Default is all of them. + keys: list of keywords, among calibfinder.badfiber_keywords. Default is all of them. yaml_file: path to a specific yaml file. By default, the code will automatically find the yaml file from the environment variable DESI_SPECTRO_CALIB and the CAMERA keyword in the headers @@ -386,17 +387,16 @@ def findfile(self,key) : """ return os.path.join(self.directory,self.data[key]) - def badfibers(self,keys=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS","BADAMPFIBERS","EXCLUDEFIBERS","NEARCHARGETRAPFIBERS", "VARIABLETHRUFIBERS"]) : + def badfibers(self,keys=badfiber_keywords) : """ Args: - keys: optional, list of keywords, among BROKENFIBERS,BADCOLUMNFIBERS,LOWTRANSMISSIONFIBERS,BADAMPFIBERS,EXCLUDEFIBERS. Default is all of them. + keys: optional, list of keywords, among calibfinder.badfiber_keywords. Default is all of them. Returns: List of bad fibers from yaml file as a 1D array of intergers """ log = get_logger() fibers=[] - badfiber_keywords=["BROKENFIBERS","BADCOLUMNFIBERS","LOWTRANSMISSIONFIBERS","BADAMPFIBERS","EXCLUDEFIBERS","NEARCHARGETRAPFIBERS","VARIABLETHRUFIBERS"] for key in keys : if key not in badfiber_keywords : log.error(f"key '{key}' not in the list of valid keys for bad fibers: {validkeys}") From 92f2c9aca6827d7d985d2ab8d4bf8213c38ee6a7 Mon Sep 17 00:00:00 2001 From: Julien Guy Date: Thu, 8 Aug 2024 10:55:03 -0700 Subject: [PATCH 4/5] add FIBERSTATUS bits in assemble_fibermap by parsing the DESI_SPECTRO_CALIB configs of all cameras using the calibfinder.badfiber function and the raw fits file headers --- py/desispec/io/fibermap.py | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/py/desispec/io/fibermap.py b/py/desispec/io/fibermap.py index 6289e886e..ee40a9c68 100644 --- a/py/desispec/io/fibermap.py +++ b/py/desispec/io/fibermap.py @@ -26,6 +26,7 @@ from desispec.io.util import (fitsheader, write_bintable, makepath, addkeys, parse_badamps, checkgzip) from desispec.io.meta import rawdata_root, findfile +from desispec.calibfinder import CalibFinder from . import iotime from .table import read_table @@ -1103,6 +1104,52 @@ def assemble_fibermap(night, expid, badamps=None, badfibers_filename=None, badfibermask = np.bitwise_or.reduce(table["FIBERSTATUS"][table["FIBER"]==fiber]) fibermap['FIBERSTATUS'][fibermap["FIBER"]==fiber] |= badfibermask + #- Add FIBERSTATUS bits based on DESI_SPECTRO_CALIB entries for all cameras + badfibers_keywords_and_maskbits={"NEARCHARGETRAPFIBERS":fibermask.NEARCHARGETRAP,"VARIABLETHRUFIBERS":fibermask.VARIABLETHRU} + badfibers=dict() + for key in badfibers_keywords_and_maskbits.keys() : + badfibers[key]=np.empty(0,dtype=int) + #- loop over cameras to look for bad fibers in the calibration yaml files with a calibfinder + for specid in range(10) : + for arm in ["b","r","z"] : + camera="{}{}".format(arm,specid) + try: + camheader = fits.getheader(rawfile, camera, disable_image_compression=True) + except KeyError: + log.debug("No camera {} in this file".format(camera)) + continue + cfinder=CalibFinder([rawheader,camheader]) + for key in badfibers_keywords_and_maskbits.keys() : + newbadfibers = cfinder.badfibers([key]) + if newbadfibers.size > 0 : + log.debug("Adding {} bad fibers {} of camera {}".format(key,newbadfibers,camera)) + # check that they do match the fiber number of this spectrograph or throw error + # to avoid any misunderstanding + if np.any((newbadfibers=(specid+1)*500)) : + mess="Not all fibers from {} belong to spectrograph ID {}".format(newbadfibers,specid) + log.error(mess) + raise KeyError(mess) + badfibers[key] = np.append(badfibers[key],newbadfibers) + + #- remove possible duplicates + # (it can happen if a bad fiber is mentionned in several cameras of the same spectro) + for key in badfibers_keywords_and_maskbits.keys() : + if badfibers[key].size > 1 : + badfibers[key] = np.unique(badfibers[key]) + #- nice message + message="Bad fibers from DESI_SPECTRO_CALIB:" + is_empty=True + for key in badfibers_keywords_and_maskbits.keys() : + if badfibers[key].size > 0 : + message += " {}:{}".format(key,badfibers[key]) + is_empty=False + if is_empty : message += " None" + log.info(message) + #- now add the keywords to FIBERSTATUS + for key in badfibers_keywords_and_maskbits.keys() : + selection = np.in1d(fibermap["FIBER"],badfibers[key]) + fibermap["FIBERSTATUS"][selection] |= badfibers_keywords_and_maskbits[key] + #- NaN are a pain; reset to dummy values for col in [ 'FIBER_X', 'FIBER_Y', From 4e65fe364ff0b56c8191ee958b70109b0056037d Mon Sep 17 00:00:00 2001 From: Julien Guy Date: Thu, 8 Aug 2024 11:21:48 -0700 Subject: [PATCH 5/5] doc/changes --- doc/changes.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/changes.rst b/doc/changes.rst index 4e80d2001..2d527c0c0 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -16,6 +16,9 @@ desispec Change Log * zproc requires exposure-qa files for tileqa step (PR `#2306`_). * Don't set envs in desispec.module that are now set in desimodules (PR `#2310`_). +* New FIBERSTATUS NEARCHARGETRAP and VARIABLETHRU set in + desispec.io.fibermap.assemble_fibermap based on content + of DESI_SPECTRO_CALIB yaml files (PR `#2313`_). .. _`#2290`: https://github.com/desihub/desispec/pull/2290 .. _`#2294`: https://github.com/desihub/desispec/pull/2294 @@ -23,6 +26,8 @@ desispec Change Log .. _`#2302`: https://github.com/desihub/desispec/pull/2302 .. _`#2306`: https://github.com/desihub/desispec/pull/2306 .. _`#2310`: https://github.com/desihub/desispec/pull/2310 +.. _`#2313`: https://github.com/desihub/desispec/pull/2313 + 0.64.0 (2024-07-01) -------------------