From 032f2b72b69da18c31eabde783322374bf8205e7 Mon Sep 17 00:00:00 2001 From: Dan Higham Date: Wed, 16 Mar 2016 14:10:19 -0700 Subject: [PATCH] Retry a maxmimum of five times for deploys and errands --- install_steps/deploy.py | 6 +++++- install_steps/run_errands.py | 13 ++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/install_steps/deploy.py b/install_steps/deploy.py index 0eb623f..6a06917 100644 --- a/install_steps/deploy.py +++ b/install_steps/deploy.py @@ -24,7 +24,11 @@ def do_step(context): task = client.wait_for_task(task_id) - while task['state'] == 'error': + retries = 0 + + while task['state'] == 'error' and retries < 5: + + retries += 1 print "Retrying deploy for {0}/manifests/{1}...".format(home_dir, m['file']) diff --git a/install_steps/run_errands.py b/install_steps/run_errands.py index c80b51a..b8f0563 100644 --- a/install_steps/run_errands.py +++ b/install_steps/run_errands.py @@ -23,7 +23,18 @@ def do_step(context): print "Running errand {0}".format(errand) task_id = client.run_errand(m['deployment-name'], errand) - client.wait_for_task(task_id) + task = client.wait_for_task(task_id) + + retries = 0 + + while task['state'] == 'error' and retries < 5: + + retries += 1 + + print "Retrying errand {0}".format(errand) + + task_id = client.run_errand(m['deployment-name'], errand) + task = client.wait_for_task(task_id) result = client.get_task_result(task_id) print "Errand finished with exit code {0}".format(result['exit_code'])