improve squeue handling and add zs by default to submit_prod

py/desispec/scripts/submit_prod.py

@@ -145,8 +145,8 @@ def submit_production(production_yaml, dry_run_level=False):
         thru_night = last_night
         log.warning(f"Setting thru_night to last_night: {thru_night}")
 
-    ## note that None defaults to "cumulative" when handed to desi_proc_night
-    ## unless no_redshifts is True
+    ## If not specified, run "cumulative" redshifts, otherwise do
+    ## as directed
     no_redshifts = False
     if 'Z_SUBMIT_TYPES' in conf:
         z_submit_types_str = str(conf['Z_SUBMIT_TYPES'])
@@ -157,7 +157,7 @@ def submit_production(production_yaml, dry_run_level=False):
             z_submit_types = [ztype.strip().lower() for ztype in
                                    z_submit_types_str.split(',')]
     else:
-        z_submit_types = None
+        z_submit_types = ['cumulative']
 
     if 'SURVEYS' in conf:
         surveys_str = str(conf['SURVEYS'])

py/desispec/workflow/queue.py

@@ -4,6 +4,7 @@
 
 """
 import os
+import re
 import numpy as np
 from astropy.table import Table, vstack
 import subprocess
@@ -518,7 +519,23 @@ def get_jobs_in_queue(user=None, include_scron=False, dry_run_level=0):
         log.critical(msg)
         raise RuntimeError(msg)
 
-    queue_info_table = Table.read(table_as_string, format='ascii.csv')
+    ## remove extra quotes that astropy table does't like
+    table_as_string = table_as_string.replace('"','')
+
+    ## remove parenthesis are also not very desirable
+    table_as_string = table_as_string.replace('(', '').replace(')', '')
+
+
+    ## remove node list with hyphen or comma otherwise it will break table reader
+    table_as_string = re.sub(r"nid\[[0-9,-]*\]", "multiple nodes", table_as_string)
+
+    try:
+        queue_info_table = Table.read(table_as_string, format='ascii.csv')
+    except:
+        log.info("Table retured by squeue couldn't be parsed. The string was:")
+        print(table_as_string)
+        raise
+
     for col in queue_info_table.colnames:
         queue_info_table.rename_column(col, col.upper())
 
@@ -559,4 +576,4 @@ def check_queue_count(user=None, include_scron=False, dry_run_level=0):
         scron entries depending on include_scron).
     """
     return len(get_jobs_in_queue(user=user, include_scron=include_scron,
-                                 dry_run_level=dry_run_level))
+                                 dry_run_level=dry_run_level))