Adding a retry to read json file on EFS

int.env

@@ -1,6 +1,6 @@
 PRINT_ENV=int
 RANCHER_LABEL=int
-IMAGE_TAG=r181113_80ed7b9
+IMAGE_TAG=r181120_6ec0d0f
 NGINX_PORT=8009
 TOMCAT_PORT=8011
 WSGI_PORT=8010

print3/main.py

@@ -12,6 +12,7 @@
 import random
 from urlparse import urlsplit
 from urllib import urlencode
+from retrying import retry
 
 
 from PyPDF2 import PdfFileMerger
@@ -69,8 +70,8 @@ def get_tomcat_backend_info():
     url = 'http:%s/%s' % (TOMCAT_LOCAL_SERVER_URL,
                           'service-print-main/checker')
     r = requests.get(url,
-                        headers={'Referer': REFERER_URL},
-                        verify=False)
+                     headers={'Referer': REFERER_URL},
+                     verify=False)
     return r.content
 
 
@@ -101,6 +102,28 @@ def print_cancel():
     return Response(status=200)
 
 
+@retry(
+    wait_exponential_multiplier=200,
+    wait_exponential_max=1000,
+    stop_max_delay=3000)
+def _read_json(filename):
+    data = None
+    try:
+        with open(filename, 'r') as data_file:
+            raw_data = data_file.read()
+            data = json.loads(raw_data)
+    except IOError:
+        raise Exception('Cannot read file {}'.format(filename))
+    except ValueError:
+        raise Exception(
+            'Cannot decode file: {} with content: {}'.format(
+                filename, raw_data))
+    except:
+        raise Exception('Unexpected error while reading {}'.format(filename))
+
+    return data
+
+
 @app.route('/printprogress')
 def print_progress():
 
@@ -111,8 +134,9 @@ def print_progress():
     if not os.path.isfile(filename):
         abort(400, '%s does not exists' % filename)
 
-    with open(filename, 'r') as data_file:
-        data = json.load(data_file)
+    data = _read_json(filename)
+    if data is None:
+        abort(500, 'Cannot read/decode {}'.format(filename))
 
     # When file is written, get current size
     if os.path.isfile(pdffile):
@@ -274,7 +298,7 @@ def worker(job):
         multi_logger.error('[Worker] Unknown exception: %s', e)
         with open(infofile, 'w+') as outfile:
             json.dump({'status': 'failed', 'done': 0,
-                   'total': 0}, outfile)
+                       'total': 0}, outfile)
 
     return (timestamp, None)
 
@@ -470,7 +494,8 @@ def write_info():
             if pdf[1] is not None:
                 pdfs[i] = pdf
             else:
-                logger.error('Retry of partial PDF also failed. Cannot merge PDF')
+                logger.error(
+                    'Retry of partial PDF also failed. Cannot merge PDF')
                 logger.error('spec: {}'.format(job))
                 return 2
 

prod.env

@@ -1,6 +1,6 @@
 PRINT_ENV=prod
 RANCHER_LABEL=prod
-IMAGE_TAG=r181113_80ed7b9
+IMAGE_TAG=r181120_6ec0d0f
 NGINX_PORT=8009
 TOMCAT_PORT=8011
 WSGI_PORT=8010

requirements.txt

@@ -14,3 +14,4 @@ Flask==0.12.1
 gevent==1.2.2
 gunicorn==19.7.1
 requests==2.20.0
+retrying==1.3.3