Set a timeout for mobile app incident proxy requests (#4306)

engine/apps/mobile_app/tests/test_mobile_app_gateway.py

@@ -8,7 +8,7 @@
 from rest_framework.test import APIClient
 from rest_framework.views import APIView
 
-from apps.mobile_app.views import MobileAppGatewayView
+from apps.mobile_app.views import PROXY_REQUESTS_TIMEOUT, MobileAppGatewayView
 from common.cloud_auth_api.client import CloudAuthApiClient, CloudAuthApiException
 
 DOWNSTREAM_BACKEND = "incident"
@@ -64,6 +64,7 @@ def test_mobile_app_gateway_properly_proxies_paths(
         data=b"",
         params={},
         headers=MOCK_DOWNSTREAM_HEADERS,
+        timeout=PROXY_REQUESTS_TIMEOUT,
     )
 
 
@@ -118,6 +119,7 @@ def test_mobile_app_gateway_proxies_query_params(
         data=b"",
         params={"foo": "bar", "baz": "hello"},
         headers=MOCK_DOWNSTREAM_HEADERS,
+        timeout=PROXY_REQUESTS_TIMEOUT,
     )
 
 
@@ -161,6 +163,7 @@ def test_mobile_app_gateway_properly_proxies_request_body(
         data=data.encode("utf-8"),
         params={},
         headers=MOCK_DOWNSTREAM_HEADERS,
+        timeout=PROXY_REQUESTS_TIMEOUT,
     )
 
 
@@ -206,7 +209,7 @@ def test_mobile_app_gateway_supported_downstream_backends(
         (requests.exceptions.ConnectionError, (), status.HTTP_502_BAD_GATEWAY),
         (requests.exceptions.HTTPError, (), status.HTTP_502_BAD_GATEWAY),
         (requests.exceptions.TooManyRedirects, (), status.HTTP_502_BAD_GATEWAY),
-        (requests.exceptions.Timeout, (), status.HTTP_502_BAD_GATEWAY),
+        (requests.exceptions.Timeout, (), status.HTTP_504_GATEWAY_TIMEOUT),
         (requests.exceptions.JSONDecodeError, ("", "", 5), status.HTTP_400_BAD_REQUEST),
         (CloudAuthApiException, (403, "http://example.com"), status.HTTP_502_BAD_GATEWAY),
     ],
@@ -317,6 +320,7 @@ def test_mobile_app_gateway_proxies_headers(
             "Authorization": f"Bearer {MOCK_AUTH_TOKEN}",
             "Content-Type": content_type_header,
         },
+        timeout=PROXY_REQUESTS_TIMEOUT,
     )
 
 

engine/apps/mobile_app/views.py

@@ -1,5 +1,6 @@
 import enum
 import logging
+import time
 import typing
 
 import requests
@@ -22,6 +23,9 @@
     from apps.user_management.models import Organization, User
 
 
+PROXY_REQUESTS_TIMEOUT = 5
+
+
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
@@ -188,6 +192,7 @@ def _get_downstream_url(
         return f"{downstream_url}/{downstream_path}"
 
     def _proxy_request(self, request: Request, *args, **kwargs) -> Response:
+        request_start = time.perf_counter()
         downstream_backend = kwargs["downstream_backend"]
         downstream_path = kwargs["downstream_path"]
         method = request.method
@@ -209,17 +214,21 @@ def _proxy_request(self, request: Request, *args, **kwargs) -> Response:
                 data=request.body,
                 params=request.query_params.dict(),
                 headers=self._get_downstream_headers(request, downstream_backend, user),
+                timeout=PROXY_REQUESTS_TIMEOUT,  # set a timeout to prevent hanging
             )
-
+            final_status = downstream_response.status_code
             logger.info(f"Successfully proxied {log_msg_common}")
             return Response(status=downstream_response.status_code, data=downstream_response.json())
         except (
             requests.exceptions.RequestException,
             requests.exceptions.JSONDecodeError,
+            requests.exceptions.Timeout,
             CloudAuthApiException,
         ) as e:
             if isinstance(e, requests.exceptions.JSONDecodeError):
                 final_status = status.HTTP_400_BAD_REQUEST
+            elif isinstance(e, requests.exceptions.Timeout):
+                final_status = status.HTTP_504_GATEWAY_TIMEOUT
             else:
                 final_status = status.HTTP_502_BAD_GATEWAY
 
@@ -235,6 +244,14 @@ def _proxy_request(self, request: Request, *args, **kwargs) -> Response:
                 exc_info=True,
             )
             return Response(status=final_status)
+        finally:
+            request_end = time.perf_counter()
+            seconds = request_end - request_start
+            logging.info(
+                f"outbound latency={str(seconds)} status={final_status} "
+                f"method={method.upper()} url={downstream_url} "
+                f"slow={int(seconds > settings.SLOW_THRESHOLD_SECONDS)} "
+            )
 
 
 """