fix element color in dash and other small tweaks

py/desispec/scripts/procdashboard.py

@@ -131,7 +131,7 @@ def main(args=None):
                 night_json_info = read_json(filename_json=filename_json)
 
             ## get the per exposure info for a night
-            night_info = populate_night_info(night, args.check_on_disk,
+            night_info = populate_night_info(night, check_on_disk=args.check_on_disk,
                                              night_json_info=night_json_info,
                                              skipd_expids=skipd_expids)
             nightly_tables[night] = night_info.copy()
@@ -236,13 +236,16 @@ def populate_night_info(night, check_on_disk=False,
         for prow in caljobs_ptab:
             jobdesc = prow['JOBDESC']
             expids = prow['EXPID']
-            lastexpid = expids[-1]
-            if lastexpid in exptab['EXPID']:
-                joint_erow = table_row_to_dict(exptab[exptab['EXPID']==lastexpid][0])
+            if jobdesc == 'ccdcalib':
+                expid = expids[0]
+            else:
+                expid = expids[-1]
+            if expid in exptab['EXPID']:
+                joint_erow = table_row_to_dict(exptab[exptab['EXPID']==expid][0])
                 joint_erow['OBSTYPE'] = jobdesc
                 joint_erow['ORDER'] = joint_erow['ORDER']+1
-                if len(expids) == 1:
-                    joint_erow['COMMENTS'] = [f"Exposure {expids[0]}"]
+                if len(expids) < 5:
+                    joint_erow['COMMENTS'] = [f"Exposure(s) {','.join(np.array(expids).astype(str))}"]
                 else:
                     joint_erow['COMMENTS'] = [f"Exposures {expids[0]}-{expids[-1]}"]
                 # ## Derive the appropriate PROCCAMWORD from the exposure table
@@ -472,9 +475,8 @@ def count_num_files(ftype, expid=None):
             row_color = 'OVERFULL'
 
         slurm_hlink, log_hlink = '----', '----'
-        if row_color not in ['GOOD', 'NULL'] and obstype.lower() in ['arc',
-                                                                     'flat',
-                                                                     'science']:
+        if row_color not in ['GOOD', 'NULL', 'PENDING'] \
+                and obstype.lower() in ['arc', 'flat', 'science']:
             file_head = obstype.lower()
             lognames = glob.glob(
                 logfiletemplate.format(pre=file_head, night=night,
@@ -540,17 +542,12 @@ def count_num_files(ftype, expid=None):
         rd["EXP TIME"] = str(exptime)
         rd["PROC CAMWORD"] = proccamword
         rd["PSF"] = _str_frac(nfiles['psf'], ncams * expected['psf'])
-        rd["FRAME"] = _str_frac(nfiles['frame'],
-                                        ncams * expected['frame'])
+        rd["FRAME"] = _str_frac(nfiles['frame'], ncams * expected['frame'])
         rd["FFLAT"] = _str_frac(nfiles['ff'], ncams * expected['ff'])
-        rd["SFRAME"] = _str_frac(nfiles['sframe'],
-                                        ncams * expected['sframe'])
-        rd["SKY"] = _str_frac(nfiles['sky'],
-                                        ncams * expected['sframe'])
-        rd["STD"] = _str_frac(nfiles['std'],
-                                        nspecs * expected['std'])
-        rd["CFRAME"] = _str_frac(nfiles['cframe'],
-                                        ncams * expected['cframe'])
+        rd["SFRAME"] = _str_frac(nfiles['sframe'], ncams * expected['sframe'])
+        rd["SKY"] = _str_frac(nfiles['sky'], ncams * expected['sframe'])
+        rd["STD"] = _str_frac(nfiles['std'], nspecs * expected['std'])
+        rd["CFRAME"] = _str_frac(nfiles['cframe'], ncams * expected['cframe'])
         rd["SLURM FILE"] = slurm_hlink
         rd["LOG FILE"] = log_hlink
         rd["COMMENTS"] = comments

py/desispec/scripts/zprocdashboard.py

@@ -33,6 +33,7 @@
     difference_camwords, parse_badamps, create_camword, camword_union, \
     columns_to_goodcamword, spectros_to_camword
 
+
 def parse(options):
     """
     Initialize the parser to read input
@@ -156,8 +157,9 @@ def main(args=None):
             subset_exptabs = all_exptabs[np.isin(all_exptabs['TILEID'], tiles)]
 
             ## get the per exposure info for a night
-            night_zinfo = populate_night_zinfo(night, doem, doqso,
-                                               dotileqa, args.check_on_disk,
+            night_zinfo = populate_night_zinfo(night, doem=doem, doqso=doqso,
+                                               dotileqa=dotileqa, dozmtl=dozmtl,
+                                               check_on_disk=args.check_on_disk,
                                                night_json_zinfo=night_json_zinfo,
                                                skipd_tileids=skipd_tileids,
                                                all_exptabs=subset_exptabs)
@@ -240,12 +242,19 @@ def populate_night_zinfo(night, doem=True, doqso=True, dotileqa=True, dozmtl=Tru
         ## if not asked to do them rather than setting specific redshift
         ## types to 1 as done above
         if not dotileqa:
+            log.info(f"Given {dotileqa=} so not expecting tile-qa files.")
             expected_by_type[ztype]['tile-qa'] = 0
         if not dozmtl:
+            log.info(f"Given {dozmtl=} so not expecting zmtl files.")
             expected_by_type[ztype]['zmtl'] = 0
 
+    for key, val in expected_by_type.items():
+        log.info(f"Expecting the following for type {key}: {val}")
+
     ## Determine the last filetype that is expected for each obstype
     terminal_steps = get_terminal_steps(expected_by_type)
+    for key, val in terminal_steps.items():
+        log.info(f"Expecting the following for terminal state for type {key}: {val}")
 
     ## Get non final Slurm states
     non_final_states = get_non_final_states()
@@ -414,13 +423,16 @@ def populate_night_zinfo(night, doem=True, doqso=True, dotileqa=True, dozmtl=Tru
             succinct_expid = str(row['EXPID'][0])
         else:
             str_expids = np.sort(row['EXPID']).astype(str)
-            for i in range(len(str_expids[0])):
-                ith_digit = str_expids[0][i]
-                if np.all([ith_digit == expid[i] for expid in str_expids]):
-                    succinct_expid += ith_digit
-                else:
-                    succinct_expid += f'[{str_expids[0][i:]}-{str_expids[-1][i:]}]'
-                    break
+            if len(str_expids) < 4:
+                succinct_expid = ','.join(str_expids)
+            else:
+                for i in range(len(str_expids[0])):
+                    ith_digit = str_expids[0][i]
+                    if np.all([ith_digit == expid[i] for expid in str_expids]):
+                        succinct_expid += ith_digit
+                    else:
+                        succinct_expid += f'[{str_expids[0][i:]}-{str_expids[-1][i:]}]'
+                        break
 
         obstype = str(row['OBSTYPE']).lower().strip()
 
@@ -528,7 +540,7 @@ def populate_night_zinfo(night, doem=True, doqso=True, dotileqa=True, dozmtl=Tru
 
 
         slurm_hlink, log_hlink = '----', '----'
-        if row_color not in ['GOOD', 'NULL']:
+        if row_color not in ['GOOD', 'NULL', 'PENDING']:
             templatelog = logfiletemplate.format(ztype=ztype, tileid=tileid,
                                                  night=night, zexpid=zfild_expid,
                                                  jobid='*', ext='log')

py/desispec/workflow/proc_dashboard_funcs.py

@@ -23,8 +23,10 @@
     default_obstypes_for_exptable, get_exposure_table_column_defaults, \
     get_exposure_table_pathname
 from desispec.workflow.proctable import get_processing_table_pathname
+from desispec.workflow.queue import get_non_final_states
 from desispec.workflow.tableio import load_table
 
+non_final_q_states = get_non_final_states()
 
 def get_output_dir(desi_spectro_redux, specprod, output_dir, makedir=True):
     if 'DESI_SPECTRO_DATA' not in os.environ.keys():
@@ -250,9 +252,9 @@ def return_color_profile():
     color_profile = dict()
     color_profile['DEFAULT'] = {'font':'#000000' ,'background':'#ccd1d1'} # gray
     color_profile['PENDING'] = {'font': '#000000', 'background': '#FFFFFF'}  # black on white
-    color_profile['RUNNING'] = color_profile['PENDING']
-    color_profile['REQUEUED'] = color_profile['PENDING']
-    color_profile['RESIZING'] = color_profile['PENDING']
+    ## for now make all non-final states the same as pending
+    for state in non_final_q_states:
+        color_profile[state] = color_profile['PENDING']
     color_profile['NULL'] = {'font': '#34495e', 'background': '#ccd1d1'}  # gray on gray
     color_profile['GOODNULL'] = {'font': '#34495e', 'background': '#7fb3d5'}  # gray on blue
     color_profile['BAD'] = {'font':'#000000' ,'background':'#d98880'}  #  red
@@ -390,8 +392,7 @@ def generate_nightly_table_html(night_info, night, show_null):
     htmltab = r'    '
     heading = (f"Night {night}{htmltab}"
                + f"Complete: {ngood}/{n_notnull}{htmltab}"
-               + f"Incomplete: {ninter}/{n_notnull}{htmltab}"
-               + f"Failed: {nbad}/{n_notnull}{htmltab}"
+               + f"Failed: {nbad+ninter}/{n_notnull}{htmltab}"
                + f"Overfull: {nover}/{n_notnull}{htmltab}"
                + f"Pending: {npending}/{n_notnull}{htmltab}"
                + f"Running: {nrunning}/{n_notnull}{htmltab}"
@@ -600,6 +601,7 @@ def _closing_str():
     return closing
 
 def _table_row(dictionary):
+    global non_final_q_states
     idlabel = dictionary.pop('COLOR')
     # color_profile = return_color_profile()
     if dictionary["STATUS"] in ['unprocessed', 'unrecorded']:
@@ -613,7 +615,10 @@ def _table_row(dictionary):
         row_str = f'<tr style="{style_str}" id="{idlabel}">'
 
     for elem in dictionary.values():
-        if re.match('^\d+\/\d+$', elem) is not None:
+        ## If job is still running don't color things yet
+        if idlabel.upper() in non_final_q_states:
+            row_str += _table_element(elem)
+        elif re.match('^\d+\/\d+$', elem) is not None:
             strs = str(elem).split('/')
             numerator, denom = int(strs[0]), int(strs[1])
             if numerator == 0 and denom == 0: