Set manifest Lambda concurrency

lambdas/service/.chalice/config.json.template.py

@@ -34,6 +34,10 @@
             "lambda_functions": {
                 "api_handler": chalice.vpc_lambda_config(app_name),
                 service.generate_manifest.name: {
+                    # The minimum request rate enforceable by the WAF (100/5min)
+                    # is too lenient for manifest requests, so we further
+                    # restrict the manifest lambda with a low concurrency.
+                    "reserved_concurrency": 10,
                     "lambda_timeout": config.service_lambda_timeout
                 },
                 service.update_health_cache.name: {

src/azul/service/async_manifest_service.py

@@ -10,6 +10,9 @@
 )
 
 import attrs
+from chalice import (
+    TooManyRequestsError,
+)
 import msgpack
 
 from azul import (
@@ -205,6 +208,8 @@ def inspect_generation(self, token: Token) -> Token | ExecutionResult:
             elif status == 'RUNNING':
                 log.info('Execution %r is still running', execution_arn)
                 return token.next()
+            elif execution['error'] == 'Lambda.TooManyRequestsException':
+                raise TooManyRequestsError(execution['error'], execution['cause'])
             else:
                 raise GenerationFailed(status=status, output=output)
 

test/service/test_manifest_async.py

@@ -25,6 +25,9 @@
 from botocore.exceptions import (
     ClientError,
 )
+from chalice import (
+    TooManyRequestsError,
+)
 from furl import (
     furl,
 )
@@ -152,25 +155,32 @@ def test_status_running(self, _sfn):
 
     def test_status_failed(self, _sfn):
         """
-        A failed manifest job should raise a GenerationFailed
+        A failed manifest job should raise an appropriate exception
         """
         service = AsyncManifestService()
         execution_name = service.execution_name(self.generation_id, iteration=0)
-        _sfn.describe_execution.return_value = {
-            'executionArn': service.execution_arn(execution_name),
-            'stateMachineArn': service.machine_arn,
-            'name': execution_name,
-            'status': 'FAILED',
-            'startDate': datetime.datetime(2018, 11, 14, 16, 6, 53, 382000),
-            'stopDate': datetime.datetime(2018, 11, 14, 16, 6, 55, 860000),
-            'input': '{"filters": {"organ": {"is": ["lymph node"]}}}',
-        }
         token = Token(generation_id=self.generation_id,
                       iteration=0,
                       request_index=0,
                       retry_after=0)
-        with self.assertRaises(GenerationFailed):
-            service.inspect_generation(token)
+        for error, exception in [
+            ('Lambda.TooManyRequestsException', TooManyRequestsError),
+            ('', GenerationFailed),
+        ]:
+            with self.subTest(error=error):
+                _sfn.describe_execution.return_value = {
+                    'executionArn': service.execution_arn(execution_name),
+                    'stateMachineArn': service.machine_arn,
+                    'name': execution_name,
+                    'status': 'FAILED',
+                    'error': error,
+                    'cause': '',
+                    'startDate': datetime.datetime(2018, 11, 14, 16, 6, 53, 382000),
+                    'stopDate': datetime.datetime(2018, 11, 14, 16, 6, 55, 860000),
+                    'input': '{"filters": {"organ": {"is": ["lymph node"]}}}',
+                }
+                with self.assertRaises(exception):
+                    service.inspect_generation(token)
 
 
 class TestManifestController(DCP1TestCase, LocalAppTestCase):