diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py index 70c672f0e8..4afea5c8b5 100755 --- a/ci/scripts/utils/rocotostat.py +++ b/ci/scripts/utils/rocotostat.py @@ -136,7 +136,7 @@ def rocoto_statcount(rocotostat): rocotostat_output = [line.split()[0:4] for line in rocotostat_output] rocotostat_output = [line for line in rocotostat_output if len(line) != 1] - status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED'] + status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED', 'UNAVAILABLE'] rocoto_status = {} status_counts = Counter(case for sublist in rocotostat_output for case in sublist) @@ -214,9 +214,16 @@ def is_stalled(rocoto_status): elif rocoto_status['DEAD'] > 0: error_return = rocoto_status['FAIL'] + rocoto_status['DEAD'] rocoto_state = 'FAIL' - elif 'UNKNOWN' in rocoto_status: - error_return = rocoto_status['UNKNOWN'] - rocoto_state = 'UNKNOWN' + elif 'UNAVAILABLE' in rocoto_status or 'UNKNOWN' in rocoto_status: + rocoto_status = attempt_multiple_times(lambda: rocoto_statcount(rocotostat), 2, 120, ProcessError) + error_return = 0 + rocoto_state = 'RUNNING' + if 'UNAVAILABLE' in rocoto_status: + error_return = rocoto_status['UNAVAILABLE'] + rocoto_state = 'UNAVAILABLE' + if 'UNKNOWN' in rocoto_status: + error_return += rocoto_status['UNKNOWN'] + rocoto_state = 'UNKNOWN' elif is_stalled(rocoto_status): rocoto_status = attempt_multiple_times(lambda: rocoto_statcount(rocotostat), 2, 120, ProcessError) if is_stalled(rocoto_status):