From c3444b806358762eddcef6a30ed8527e580483ed Mon Sep 17 00:00:00 2001 From: kremin Date: Wed, 14 Aug 2024 22:36:54 -0700 Subject: [PATCH] bug fixes to resub and standardize cache cols --- py/desispec/workflow/exptable.py | 4 ++-- py/desispec/workflow/processing.py | 5 +---- py/desispec/workflow/proctable.py | 8 ++++---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/py/desispec/workflow/exptable.py b/py/desispec/workflow/exptable.py index ad6bf43b8..0edb399ae 100644 --- a/py/desispec/workflow/exptable.py +++ b/py/desispec/workflow/exptable.py @@ -1079,10 +1079,10 @@ def update_science_etab_cache(etab): cleaned_etab = _select_sciences_from_etab(etab) new_nights = np.unique(cleaned_etab['NIGHT']) log.info(f'Removing all current entries in science exposure ' - + f'table row cache for nights {new_nights}') + + f'table row cache for nights {list(new_nights)}') conflicting_entries = np.isin(_science_etab_cache['NIGHT'], new_nights) log.info( - f"Removing {len(conflicting_entries)} rows and adding {len(cleaned_etab)} rows " + f"Removing {np.sum(conflicting_entries)} rows and adding {len(cleaned_etab)} rows " + f"to science exposure table row cache.") keep = np.bitwise_not(conflicting_entries) _science_etab_cache = _science_etab_cache[keep] diff --git a/py/desispec/workflow/processing.py b/py/desispec/workflow/processing.py index 0a07fe5e8..5868dd4d1 100644 --- a/py/desispec/workflow/processing.py +++ b/py/desispec/workflow/processing.py @@ -1220,7 +1220,7 @@ def recursive_submit_failed(rown, proc_table, submits, id_to_row_map, ptab_name= if resubmission_states is None: resubmission_states = get_resubmission_states() ideps = proc_table['INT_DEP_IDS'][rown] - if ideps is None: + if ideps is None or len(ideps)==0: proc_table['LATEST_DEP_QID'][rown] = np.ndarray(shape=0).astype(int) else: all_valid_states = list(resubmission_states.copy()) @@ -1229,13 +1229,10 @@ def recursive_submit_failed(rown, proc_table, submits, id_to_row_map, ptab_name= othernight_idep_qid_lookup = {} for idep in np.sort(np.atleast_1d(ideps)): if idep not in id_to_row_map: - log.info(idep // 1000) - log.info(row['INTID'] // 1000) if idep // 1000 != row['INTID'] // 1000: log.info(f"Internal ID: {idep} not in id_to_row_map. " + "This is expected since it's from another day. ") reference_night = 20000000 + (idep // 1000) - log.info(reference_night) reftab = read_minimal_full_proctab_cols(nights=[reference_night]) if reftab is None: msg = f"The dependency is from night={reference_night}" \ diff --git a/py/desispec/workflow/proctable.py b/py/desispec/workflow/proctable.py index 5455ab649..3d995c66f 100644 --- a/py/desispec/workflow/proctable.py +++ b/py/desispec/workflow/proctable.py @@ -549,14 +549,14 @@ def update_tilenight_ptab_cache(ptab): log = get_logger() ## If the cache doesn't exist, don't update it. if _tilenight_ptab_cache is None: - log.debug(f'Science exptab cache does not exist, so not updating') + log.debug(f'Tilenight proctab cache does not exist, so not updating') return cleaned_ptab = _select_tilenights_from_ptab(ptab) new_nights = np.unique(cleaned_ptab['NIGHT']) log.info(f'Removing all current entries in processing table tilenight ' - + f'selection cache for nights {new_nights}') + + f'selection cache for nights {list(new_nights)}') conflicting_entries = np.isin(_tilenight_ptab_cache['NIGHT'], new_nights) - log.info(f"Removing {len(conflicting_entries)} rows and adding " + log.info(f"Removing {np.sum(conflicting_entries)} rows and adding " + f"{len(cleaned_ptab)} rows " + f"to processing table tilenight cache.") keep = np.bitwise_not(conflicting_entries) @@ -683,6 +683,6 @@ def update_full_ptab_cache(ptab): t = ptab[_required_full_ptab_cols] new_nights = np.unique(t['NIGHT']) log.info(f'Replacing all current entries in processing table ' - + f'cache for nights {new_nights.data}') + + f'cache for nights {list(new_nights)}') for night in new_nights: _full_ptab_cache[night] = t[t['NIGHT'] == night]