select calib dark with most cameras

py/desispec/test/test_workflow_calibration_selection.py

@@ -40,6 +40,7 @@ def _make_arcset_etable(self, narcsperset=5, expid_offset=0, mjd=55555.0,
 
         arcset['LASTSTEP'] = ['ignore'] * ntotalarcs
         arcset['LASTSTEP'][:] = 'all'
+        arcset['CAMWORD'] = 'a0123456789'
         arcset['BADCAMWORD'] = ['b0123456789r0123456789'] * ntotalarcs
         arcset['BADCAMWORD'][:] = ''
         arcset['BADAMPS'] = ['b0123456789r0123456789'] * ntotalarcs
@@ -82,6 +83,7 @@ def _make_flatset_etable(self, nflatsperset=3, nflatsets=4,
 
         flatset['LASTSTEP'] = ['ignore'] * nexps
         flatset['LASTSTEP'][:] = 'all'
+        flatset['CAMWORD'] = 'a0123456789'
         flatset['BADCAMWORD'] = ['b0123456789r0123456789'] * nexps
         flatset['BADCAMWORD'][:] = ''
         flatset['BADAMPS'] = ['b0123456789r0123456789'] * nexps
@@ -447,3 +449,64 @@ def test_missing_cals(self):
         self.assertEqual(len(result), 0)
         result = determine_calibrations_to_proc(badset)
         self.assertEqual(len(result), 0)
+
+    def test_dark_selection(self):
+        """
+        Test selection of which dark to use
+        """
+        from desispec.workflow.calibration_selection import \
+                determine_calibrations_to_proc
+        etable = self._make_arcflatset_etable()
+
+        # start dark EXPIDs at 100 for test comparison simplicity
+        etable.add_row(dict(
+            EXPID=100, SEQNUM=1, SEQTOT=3, LASTSTEP='all',
+            CAMWORD='a0123456789',
+            BADCAMWORD='a23', EXPTIME=300.1,
+            PROGRAM='calib dark 5min', OBSTYPE='dark'))
+
+        etable.add_row(dict(
+            EXPID=101, SEQNUM=2, SEQTOT=3, LASTSTEP='all',
+            CAMWORD='a0123456789',
+            BADCAMWORD='', EXPTIME=300.1,
+            PROGRAM='calib dark 5min', OBSTYPE='dark'))
+
+        etable.add_row(dict(
+            EXPID=102, SEQNUM=3, SEQTOT=3, LASTSTEP='all',
+            CAMWORD='a0123456789',
+            BADCAMWORD='', EXPTIME=300.1,
+            PROGRAM='calib dark 5min', OBSTYPE='dark'))
+
+        # should pick second 300s dark with BADCAMWORD='' instead of 'a23'
+        cal_etable = determine_calibrations_to_proc(etable)
+
+        idark = np.where(cal_etable['OBSTYPE'] == 'dark')[0][0]
+        self.assertEqual(cal_etable['EXPID'][idark], 101)
+
+
+    def test_select_calib_dark(self):
+        """Test workflow.calibration_selection.select_calib_dark"""
+        from desispec.workflow.calibration_selection import select_calib_dark
+        etable = Table()
+        etable['EXPID'] = [0,1,2,3]
+        etable['OBSTYPE'] = 'dark'
+        etable['EXPTIME'] = 300.1
+        etable['LASTSTEP'] = 'ignore'  # to get size right
+        etable['LASTSTEP'][:] = 'all'  # then reset to all good
+        etable['CAMWORD'] = 'a0123456789'
+        etable['BADCAMWORD'] = ['a234', '', '', '']
+        etable['BADAMPS'] = ''
+
+        # EXPID 0 has bad cameras, so should pick EXPID 1
+        self.assertEqual(select_calib_dark(etable), 1)
+
+        # ... but not if EXPID1 is ignored, then EXPID 2
+        etable['LASTSTEP'][1] = 'ignore'
+        self.assertEqual(select_calib_dark(etable), 2)
+
+        # ... but not if EXPID 2 has more bad cameras than EXPID 3
+        etable['BADCAMWORD'][2] = 'a78'
+        etable['BADCAMWORD'][3] = 'a1'
+        self.assertEqual(select_calib_dark(etable), 3)
+
+

py/desispec/workflow/calibration_selection.py

@@ -8,7 +8,48 @@
 from collections import Counter
 
 from desiutil.log import get_logger
-from desispec.io.util import decode_camword, parse_badamps, all_impacted_cameras
+from desispec.io.util import decode_camword, parse_badamps, all_impacted_cameras, erow_to_goodcamword
+
+
+def select_calib_dark(etable):
+    """
+    Returns index of first dark with most available cameras for badcolumn calibration
+
+    Args:
+        etable (astropy.table.Table): a DESI exposure_table
+
+    Returns:
+        index (int) of which row to use
+
+    Raises ValueError if no good darks are available.
+    """
+    # make copy so we can add utility columns and sort without modifying original
+    etable = etable.copy()
+
+    # add ROW so that after filtering and sorting we can still get original index
+    etable['ROW'] = np.arange(len(etable))
+
+    keep = np.where((etable['OBSTYPE']=='dark') & (etable['LASTSTEP'] != 'ignore'))[0]
+    if len(keep) == 0:
+        raise ValueError('No good dark exposures exptime>295 found in etable')
+
+    etable = etable[keep]
+
+    # count good cameras per row
+    etable['NUM_GOODCAM'] = 0
+    for i in range(len(etable)):
+        cameras = decode_camword(erow_to_goodcamword(etable[i]))
+        etable['NUM_GOODCAM'][i] = len(cameras)
+
+    etable['NUM_MISSING_CAMERAS'] = np.max(etable['NUM_GOODCAM']) - etable['NUM_GOODCAM']
+
+    # sort by decreasing number of missing cameras and increasing exposure time
+    etable.sort( ('NUM_MISSING_CAMERAS', 'EXPID') )
+
+    print(etable)
+
+    # original row index of first entry should be best
+    return etable['ROW'][0]
 
 
 def determine_calibrations_to_proc(etable, do_cte_flats=True,
@@ -79,10 +120,11 @@ def determine_calibrations_to_proc(etable, do_cte_flats=True,
     ## does it's own selection
     best_arcflat_set = find_best_arc_flat_sets(cal_etable)
 
-    ## Create the output table with all zeros, the first valid dark,
+    ## Create the output table with all zeros, the best selected dark,
     ## the best set of arcs and flats, and all cte flats
     zeros = valid_etable[exptypes=='zero']
-    out_table = vstack([zeros, valid_etable[exptypes == 'dark'][:1]])
+    idark = select_calib_dark(valid_etable)
+    out_table = vstack([zeros, valid_etable[idark:idark+1]])
     out_table = vstack([out_table, best_arcflat_set])
 
     ## If doing cte flats, select one of each exptime based on proximity to the