update colors based on status in dash

py/desispec/scripts/procdashboard.py

@@ -25,7 +25,7 @@
     get_terminal_steps, get_tables
 from desispec.workflow.proctable import get_processing_table_pathname, \
     table_row_to_dict
-from desispec.workflow.queue import update_from_queue
+from desispec.workflow.queue import update_from_queue, get_non_final_states
 from desispec.workflow.tableio import load_table
 from desispec.io.meta import specprod_root, rawdata_root
 from desispec.io.util import decode_camword, camword_to_spectros, \
@@ -193,6 +193,9 @@ def populate_night_info(night, check_on_disk=False,
     ## Determine the last filetype that is expected for each obstype
     terminal_steps = get_terminal_steps(expected_by_type)
 
+    ## Get non final Slurm states
+    non_final_states = get_non_final_states()
+
     specproddir = specprod_root()
     webpage = os.environ['DESI_DASHBOARD']
 
@@ -445,10 +448,19 @@ def count_num_files(ftype, expid=None):
         else:
             nexpected = ncams
 
+        if expid in expid_processing:
+            status = row['STATUS']
+        elif expid in unaccounted_for_expids:
+            status = 'unaccounted'
+        else:
+            status = 'unprocessed'
+
         if terminal_step is None:
             row_color = 'NULL'
         elif expected[terminal_step] == 0:
             row_color = 'NULL'
+        elif status in non_final_states:
+            row_color = status
         elif nfiles[terminal_step] == 0:
             row_color = 'BAD'
         elif nfiles[terminal_step] < nexpected:
@@ -458,13 +470,6 @@ def count_num_files(ftype, expid=None):
         else:
             row_color = 'OVERFULL'
 
-        if expid in expid_processing:
-            status = row['STATUS']
-        elif expid in unaccounted_for_expids:
-            status = 'unaccounted'
-        else:
-            status = 'unprocessed'
-
         slurm_hlink, log_hlink = '----', '----'
         if row_color not in ['GOOD', 'NULL'] and obstype.lower() in ['arc',
                                                                      'flat',

py/desispec/scripts/zprocdashboard.py

@@ -14,7 +14,7 @@
 from os import listdir
 import json
 
-from desispec.workflow.queue import update_from_queue
+from desispec.workflow.queue import update_from_queue, get_non_final_states
 # import desispec.io.util
 from desiutil.log import get_logger
 from desispec.workflow.exptable import get_exposure_table_pathname, \
@@ -247,6 +247,9 @@ def populate_night_zinfo(night, doem=True, doqso=True, dotileqa=True, dozmtl=Tru
     ## Determine the last filetype that is expected for each obstype
     terminal_steps = get_terminal_steps(expected_by_type)
 
+    ## Get non final Slurm states
+    non_final_states = get_non_final_states()
+
     specproddir = specprod_root()
     webpage = os.environ['DESI_DASHBOARD']
     logpath = os.path.join(specproddir, 'run', 'scripts', 'tiles')
@@ -500,10 +503,19 @@ def populate_night_zinfo(night, doem=True, doqso=True, dotileqa=True, dozmtl=Tru
             elif terminal_step == 'em':
                 true_terminal_step = 'emline'
 
+        if unique_key in uniqs_processing:
+            status = row['STATUS']
+        elif unique_key in unaccounted_for_tileids:
+            status = 'unrecorded'
+        else:
+            status = 'unprocessed'
+
         if true_terminal_step is None:
             row_color = 'NULL'
         elif expected[terminal_step] == 0:
             row_color = 'NULL'
+        elif status in non_final_states:
+            row_color = status
         elif nfiles[true_terminal_step] == 0:
             row_color = 'BAD'
         elif nfiles[true_terminal_step] < npossible:
@@ -513,12 +525,7 @@ def populate_night_zinfo(night, doem=True, doqso=True, dotileqa=True, dozmtl=Tru
         else:
             row_color = 'OVERFULL'
 
-        if unique_key in uniqs_processing:
-            status = row['STATUS']
-        elif unique_key in unaccounted_for_tileids:
-            status = 'unrecorded'
-        else:
-            status = 'unprocessed'
+
 
         slurm_hlink, log_hlink = '----', '----'
         if row_color not in ['GOOD', 'NULL']:

py/desispec/workflow/proc_dashboard_funcs.py

@@ -249,7 +249,10 @@ def check_running(proc_name= 'desi_dailyproc',suppress_outputs=False):
 def return_color_profile():
     color_profile = dict()
     color_profile['DEFAULT'] = {'font':'#000000' ,'background':'#ccd1d1'} # gray
-    color_profile['PENDING'] = {'font': '#000000', 'background': '#ccd1d1'}  # gray
+    color_profile['PENDING'] = {'font': '#000000', 'background': '#FFFFFF'}  # black on white
+    color_profile['RUNNING'] = color_profile['PENDING']
+    color_profile['REQUEUED'] = color_profile['PENDING']
+    color_profile['RESIZING'] = color_profile['PENDING']
     color_profile['NULL'] = {'font': '#34495e', 'background': '#ccd1d1'}  # gray on gray
     color_profile['GOODNULL'] = {'font': '#34495e', 'background': '#7fb3d5'}  # gray on blue
     color_profile['BAD'] = {'font':'#000000' ,'background':'#d98880'}  #  red
@@ -342,6 +345,7 @@ def generate_nightly_table_html(night_info, night, show_null):
 
     ngood, ninter, nbad, nnull, nover, n_notnull, noprocess, norecord = \
         0, 0, 0, 0, 0, 0, 0, 0
+    npending, nrunning = 0, 0
 
     main_body = ""
     for key, row_info in reversed(night_info.items()):
@@ -350,7 +354,7 @@ def generate_nightly_table_html(night_info, night, show_null):
             continue
         main_body += ("\t" + table_row + "\n")
         status = str(row_info["STATUS"]).lower()
-        if status == 'processing':
+        if status not in ['unprocessed', 'unrecorded']:
             if 'COLOR' in row_info:
                 color = row_info['COLOR']
             else:
@@ -367,6 +371,12 @@ def generate_nightly_table_html(night_info, night, show_null):
             elif color == 'OVERFULL':
                 nover += 1
                 n_notnull += 1
+            elif color == 'PENDING':
+                npending += 1
+                n_notnull += 1
+            elif color == 'RUNNING':
+                nrunning += 1
+                n_notnull += 1
             else:
                 nnull += 1
         elif status == 'unprocessed':
@@ -383,6 +393,8 @@ def generate_nightly_table_html(night_info, night, show_null):
                + f"Incomplete: {ninter}/{n_notnull}{htmltab}"
                + f"Failed: {nbad}/{n_notnull}{htmltab}"
                + f"Overfull: {nover}/{n_notnull}{htmltab}"
+               + f"Pending: {npending}/{n_notnull}{htmltab}"
+               + f"Running: {nrunning}/{n_notnull}{htmltab}"
                + f"Unprocessed: {noprocess}{htmltab}"
                + f"NoTabEntry: {norecord}{htmltab}"
                + f"Other: {nnull}"