Detect when Cirrus job runs out of CI minutes

README.md

@@ -88,6 +88,12 @@ optional arguments:
                         marker per line. If any marker is found in Cirrus CI
                         output for a failed build, the build is retried once
                         more. Default: $CIRRUS_FLAKY_MARKERS_FILE
+
+return values:
+  - 0: The cirrus job was successful
+  - 1: The job failed
+  - 2: Error in 'cirrus-run'
+  - 3: The Cirrus CI job ran out of CI minutes
 ```
 
 

cirrus_run/cli.py

@@ -14,7 +14,7 @@
 
 from . import CirrusAPI
 from .throbber import ProgressBar
-from .queries import build_log, get_repo, create_build, wait_build, CirrusBuildError
+from .queries import build_log, get_repo, create_build, wait_build, CirrusBuildError, CirrusCreditsError
 
 log = logging.getLogger(__name__)
 
@@ -47,10 +47,12 @@ def run(args, retry_index=0):
     print('Build created: {}'.format(build_url))
     with ProgressBar('' if args.verbose else '.'):
         try:
-            wait_build(api, build_id, abort=args.timeout*60)
+            wait_build(api, build_id, abort=args.timeout*60, credits_error_message=args.cirrus_out_of_ci_credits_message)
             rc, status, message = 0, 'successful', ''
         except CirrusBuildError:
             rc, status, message = 1, 'failed', ''
+        except CirrusCreditsError:
+            rc, status, message = 3, 'error', 'Out of CI credits'
         except Exception as exc:
             rc, status, message = 2, 'error', '{exception}: {text}'.format(
                                         exception=exc.__class__.__name__,
@@ -224,6 +226,15 @@ def parse_args(*a, **ka):
             'the build is retried once more. Default: ${}'
         ).format(ENVIRONMENT['flaky_markers']),
     )
+    parser.add_argument(
+        '--cirrus-out-of-ci-credits-message',
+        default='Monthly compute limit exceeded',
+        help=(
+            'Error message passed via "notifications" from Cirrus CI reported'
+            'when the CI job failed due to lack of CI credits.'
+            'Default: "Monthly compute limit exceeded"'
+        ),
+    )
     args = parser.parse_args(*a, **ka)
 
     if not args.token:

cirrus_run/queries.py

@@ -22,6 +22,8 @@ class CirrusQueryError(ValueError):
 class CirrusBuildError(RuntimeError):
     '''Raised on build failures'''
 
+class CirrusCreditsError(RuntimeError):
+    '''Raised when build fails due to lack of CI credits'''
 
 class CirrusTimeoutError(RuntimeError):
     '''Raised when build takes too long'''
@@ -85,14 +87,19 @@ def create_build(api: CirrusAPI,
     return answer['createBuild']['build']['id']
 
 
-def wait_build(api, build_id: str, delay=3, abort=60*60):
+def wait_build(api, build_id: str, delay=3, abort=60*60, credits_error_message=None):
     '''Wait until build finishes'''
     ERROR_CONFIRM_TIMES = 3
 
     query = '''
         query GetBuild($build: ID!) {
             build(id: $build) {
                 status
+                tasks {
+                    notifications {
+                        message
+                    }
+                }
             }
         }
     '''
@@ -103,7 +110,7 @@ def wait_build(api, build_id: str, delay=3, abort=60*60):
     while time() < time_start + abort:
         response = api(query, params)
         status = response['build']['status']
-        log.info('build {}: {}'.format(build_id, status))
+        log.info('build https://cirrus-ci.com/build/{}: {}'.format(build_id, status))
         if status in {'COMPLETED'}:
             return True
         if status in {'CREATED', 'TRIGGERED', 'EXECUTING'}:
@@ -116,6 +123,12 @@ def wait_build(api, build_id: str, delay=3, abort=60*60):
                 sleep(2 * delay / (ERROR_CONFIRM_TIMES - 1))
                 continue
             else:
+                if credits_error_message is not None:
+                    for task in response['build']['tasks']:
+                        for notif in task['notifications']:
+                            if credits_error_message in notif['message']:
+                                raise CirrusCreditsError('build {} ran out of CI credits'.format(build_id))
+
                 raise CirrusBuildError('build {} was terminated: {}'.format(build_id, status))
         raise ValueError('build {} returned unknown status: {}'.format(build_id, status))
     raise CirrusTimeoutError('build {} timed out'.format(build_id))