diff --git a/bin/desi_run_prod b/bin/desi_submit_prod similarity index 100% rename from bin/desi_run_prod rename to bin/desi_submit_prod diff --git a/py/desispec/scripts/submit_prod.py b/py/desispec/scripts/submit_prod.py index a6902f5e7..253aefcdd 100644 --- a/py/desispec/scripts/submit_prod.py +++ b/py/desispec/scripts/submit_prod.py @@ -145,8 +145,8 @@ def submit_production(production_yaml, dry_run_level=False): thru_night = last_night log.warning(f"Setting thru_night to last_night: {thru_night}") - ## note that None defaults to "cumulative" when handed to desi_proc_night - ## unless no_redshifts is True + ## If not specified, run "cumulative" redshifts, otherwise do + ## as directed no_redshifts = False if 'Z_SUBMIT_TYPES' in conf: z_submit_types_str = str(conf['Z_SUBMIT_TYPES']) @@ -157,7 +157,7 @@ def submit_production(production_yaml, dry_run_level=False): z_submit_types = [ztype.strip().lower() for ztype in z_submit_types_str.split(',')] else: - z_submit_types = None + z_submit_types = ['cumulative'] if 'SURVEYS' in conf: surveys_str = str(conf['SURVEYS']) diff --git a/py/desispec/workflow/queue.py b/py/desispec/workflow/queue.py index b3ef09cf0..21864b5d5 100644 --- a/py/desispec/workflow/queue.py +++ b/py/desispec/workflow/queue.py @@ -4,6 +4,7 @@ """ import os +import re import numpy as np from astropy.table import Table, vstack import subprocess @@ -518,7 +519,23 @@ def get_jobs_in_queue(user=None, include_scron=False, dry_run_level=0): log.critical(msg) raise RuntimeError(msg) - queue_info_table = Table.read(table_as_string, format='ascii.csv') + ## remove extra quotes that astropy table does't like + table_as_string = table_as_string.replace('"','') + + ## remove parenthesis are also not very desirable + table_as_string = table_as_string.replace('(', '').replace(')', '') + + + ## remove node list with hyphen or comma otherwise it will break table reader + table_as_string = re.sub(r"nid\[[0-9,-]*\]", "multiple nodes", table_as_string) + + try: + queue_info_table = Table.read(table_as_string, format='ascii.csv') + except: + log.info("Table retured by squeue couldn't be parsed. The string was:") + print(table_as_string) + raise + for col in queue_info_table.colnames: queue_info_table.rename_column(col, col.upper()) @@ -559,4 +576,4 @@ def check_queue_count(user=None, include_scron=False, dry_run_level=0): scron entries depending on include_scron). """ return len(get_jobs_in_queue(user=user, include_scron=include_scron, - dry_run_level=dry_run_level)) \ No newline at end of file + dry_run_level=dry_run_level))