diff --git a/lambdas/indexer/app.py b/lambdas/indexer/app.py index 975d060f16..a84a6d617d 100644 --- a/lambdas/indexer/app.py +++ b/lambdas/indexer/app.py @@ -3,11 +3,7 @@ Optional, ) -# noinspection PyPackageRequirements import chalice -from chalice import ( - Response, -) from azul import ( CatalogName, @@ -15,14 +11,13 @@ config, ) from azul.chalice import ( - AzulChaliceApp, LambdaMetric, ) from azul.deployment import ( aws, ) from azul.health import ( - HealthController, + HealthApp, ) from azul.hmac import ( HMACAuthentication, @@ -46,9 +41,6 @@ from azul.openapi.responses import ( json_content, ) -from azul.openapi.spec import ( - CommonEndpointSpecs, -) log = logging.getLogger(__name__) @@ -64,16 +56,12 @@ # changes and reset the minor version to zero. Otherwise, increment only # the minor version for backwards compatible changes. A backwards # compatible change is one that does not require updates to clients. - 'version': '1.0' + 'version': '1.1' } } -class IndexerApp(AzulChaliceApp, SignatureHelper): - - @cached_property - def health_controller(self): - return self._controller(HealthController, lambda_name='indexer') +class IndexerApp(HealthApp, SignatureHelper): @cached_property def index_controller(self) -> IndexController: @@ -98,7 +86,9 @@ def log_forwarder(self, prefix: str): error_decorator = self.metric_alarm(metric=LambdaMetric.errors, threshold=1, # One alarm … period=24 * 60 * 60) # … per day. - throttle_decorator = self.metric_alarm(metric=LambdaMetric.throttles) + throttle_decorator = self.metric_alarm(metric=LambdaMetric.throttles, + threshold=0, + period=5 * 60) retry_decorator = self.retry(num_retries=2) def decorator(f): @@ -115,120 +105,7 @@ def _authenticate(self) -> Optional[HMACAuthentication]: app = IndexerApp() configure_app_logging(app, log) - -@app.route( - '/', - cache_control='public, max-age=0, must-revalidate', - cors=False -) -def swagger_ui(): - return app.swagger_ui() - - -@app.route( - '/static/{file}', - cache_control='public, max-age=86400', - cors=True -) -def static_resource(file): - return app.swagger_resource(file) - - -common_specs = CommonEndpointSpecs(app_name='indexer') - - -@app.route( - '/openapi', - methods=['GET'], - cache_control='public, max-age=500', - cors=True, - **common_specs.openapi -) -def openapi(): - return Response(status_code=200, - headers={'content-type': 'application/json'}, - body=app.spec()) - - -@app.route( - '/version', - methods=['GET'], - cors=True, - **common_specs.version -) -def version(): - from azul.changelog import ( - compact_changes, - ) - return { - 'git': config.lambda_git_status, - 'changes': compact_changes(limit=10) - } - - -@app.route( - '/health', - methods=['GET'], - cors=True, - **common_specs.full_health -) -def health(): - return app.health_controller.health() - - -@app.route( - '/health/basic', - methods=['GET'], - cors=True, - **common_specs.basic_health -) -def basic_health(): - return app.health_controller.basic_health() - - -@app.route( - '/health/cached', - methods=['GET'], - cors=True, - **common_specs.cached_health -) -def cached_health(): - return app.health_controller.cached_health() - - -@app.route( - '/health/fast', - methods=['GET'], - cors=True, - **common_specs.fast_health -) -def fast_health(): - return app.health_controller.fast_health() - - -@app.route( - '/health/{keys}', - methods=['GET'], - cors=True, - **common_specs.custom_health -) -def health_by_key(keys: Optional[str] = None): - return app.health_controller.custom_health(keys) - - -@app.metric_alarm(metric=LambdaMetric.errors, - threshold=1, - period=24 * 60 * 60) -@app.metric_alarm(metric=LambdaMetric.throttles) -@app.retry(num_retries=0) -# FIXME: Remove redundant prefix from name -# https://github.com/DataBiosphere/azul/issues/5337 -@app.schedule( - 'rate(1 minute)', - name='indexercachehealth' -) -def update_health_cache(_event: chalice.app.CloudWatchEvent): - app.health_controller.update_cache() +globals().update(app.default_routes()) @app.route( @@ -303,9 +180,11 @@ def post_notification(catalog: CatalogName, action: str): @app.metric_alarm(metric=LambdaMetric.errors, - threshold=int(config.contribution_concurrency(retry=False) * 2 / 3)) + threshold=int(config.contribution_concurrency(retry=False) * 2 / 3), + period=5 * 60) @app.metric_alarm(metric=LambdaMetric.throttles, - threshold=int(96000 / config.contribution_concurrency(retry=False))) + threshold=int(96000 / config.contribution_concurrency(retry=False)), + period=5 * 60) @app.on_sqs_message( queue=config.notifications_queue_name(), batch_size=1 @@ -315,9 +194,11 @@ def contribute(event: chalice.app.SQSEvent): @app.metric_alarm(metric=LambdaMetric.errors, - threshold=int(config.aggregation_concurrency(retry=False) * 3)) + threshold=int(config.aggregation_concurrency(retry=False) * 3), + period=5 * 60) @app.metric_alarm(metric=LambdaMetric.throttles, - threshold=int(37760 / config.aggregation_concurrency(retry=False))) + threshold=int(37760 / config.aggregation_concurrency(retry=False)), + period=5 * 60) @app.on_sqs_message( queue=config.tallies_queue_name(), batch_size=IndexController.document_batch_size @@ -330,8 +211,11 @@ def aggregate(event: chalice.app.SQSEvent): # with more RAM in the tallies_retry queue. @app.metric_alarm(metric=LambdaMetric.errors, - threshold=int(config.aggregation_concurrency(retry=True) * 1 / 16)) -@app.metric_alarm(metric=LambdaMetric.throttles) + threshold=int(config.aggregation_concurrency(retry=True) * 1 / 16), + period=5 * 60) +@app.metric_alarm(metric=LambdaMetric.throttles, + threshold=0, + period=5 * 60) @app.on_sqs_message( queue=config.tallies_queue_name(retry=True), batch_size=IndexController.document_batch_size @@ -344,9 +228,11 @@ def aggregate_retry(event: chalice.app.SQSEvent): # retried with more RAM and a longer timeout in the notifications_retry queue. @app.metric_alarm(metric=LambdaMetric.errors, - threshold=int(config.contribution_concurrency(retry=True) * 1 / 4)) + threshold=int(config.contribution_concurrency(retry=True) * 1 / 4), + period=5 * 60) @app.metric_alarm(metric=LambdaMetric.throttles, - threshold=int(31760 / config.contribution_concurrency(retry=True))) + threshold=int(31760 / config.contribution_concurrency(retry=True)), + period=5 * 60) @app.on_sqs_message( queue=config.notifications_queue_name(retry=True), batch_size=1 diff --git a/lambdas/indexer/openapi.json b/lambdas/indexer/openapi.json index 7cc35b914b..22cf58bc2e 100644 --- a/lambdas/indexer/openapi.json +++ b/lambdas/indexer/openapi.json @@ -1,11 +1,27 @@ { "openapi": "3.0.1", "info": { - "title": "azul_indexer", + "title": "azul-indexer-dev", "description": "\nThis is the internal API for Azul's indexer component.\n", - "version": "1.0" + "version": "1.1" }, "paths": { + "/": { + "get": { + "summary": "A Swagger UI for interactive use of this REST API", + "tags": [ + "Auxiliary" + ], + "responses": { + "200": { + "description": "The response body is an HTML page containing the Swagger UI" + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" + } + } + } + }, "/openapi": { "get": { "summary": "Return OpenAPI specifications for this REST API", @@ -59,6 +75,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -66,6 +85,36 @@ ] } }, + "/static/{file}": { + "parameters": [ + { + "name": "file", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "The name of a static file to be returned" + } + ], + "get": { + "summary": "Static files needed for the Swagger UI", + "tags": [ + "Auxiliary" + ], + "responses": { + "200": { + "description": "The response body is the contents of the requested file" + }, + "404": { + "description": "The requested file does not exist" + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" + } + } + } + }, "/version": { "get": { "summary": "Describe current version of this REST API", @@ -137,6 +186,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -233,6 +285,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -322,6 +377,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -417,6 +475,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -512,6 +573,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -632,6 +696,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -741,6 +808,9 @@ }, "401": { "description": "Request lacked a valid HMAC header" + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } diff --git a/lambdas/layer/app.py b/lambdas/layer/app.py index 5923e1f3c7..f0370973cb 100644 --- a/lambdas/layer/app.py +++ b/lambdas/layer/app.py @@ -10,9 +10,10 @@ app = AzulChaliceApp(app_name=config.qualified_resource_name('dependencies'), app_module_path=__file__, - unit_test=False) + unit_test=False, + spec={}) -@app.route('/') +@app.route('/', method_spec={}) def foo(): pass diff --git a/lambdas/service/app.py b/lambdas/service/app.py index fea36acd82..5bd42e97e7 100644 --- a/lambdas/service/app.py +++ b/lambdas/service/app.py @@ -22,7 +22,6 @@ import urllib.parse import attr -import chalice from chalice import ( BadRequestError as BRE, ChaliceViewError, @@ -52,9 +51,7 @@ OAuth2, ) from azul.chalice import ( - AzulChaliceApp, C, - LambdaMetric, ) from azul.collections import ( OrderedSet, @@ -63,6 +60,7 @@ AccessMethod, ) from azul.health import ( + HealthApp, HealthController, ) from azul.indexer.document import ( @@ -80,9 +78,6 @@ responses, schema, ) -from azul.openapi.spec import ( - CommonEndpointSpecs, -) from azul.plugins import ( ManifestFormat, MetadataPlugin, @@ -232,7 +227,7 @@ # changes and reset the minor version to zero. Otherwise, increment only # the minor version for backwards compatible changes. A backwards # compatible change is one that does not require updates to clients. - 'version': '9.2' + 'version': '9.3' }, 'tags': [ { @@ -271,12 +266,18 @@ 'description': fd(''' Describes various aspects of the Azul service ''') + }, + { + 'name': 'Deprecated', + 'description': fd(''' + Endpoints that should not be used and that will be removed + ''') } ] } -class ServiceApp(AzulChaliceApp): +class ServiceApp(HealthApp): def spec(self) -> JSON: return { @@ -312,7 +313,8 @@ def drs_controller(self) -> DRSController: @cached_property def health_controller(self) -> HealthController: - return self._controller(HealthController, lambda_name='service') + return self._controller(HealthController, + lambda_name=self.unqualified_app_name) @cached_property def catalog_controller(self) -> CatalogController: @@ -463,29 +465,26 @@ def manifest_url(self, app = ServiceApp() configure_app_logging(app, log) - -@app.route( - '/', - cache_control='public, max-age=0, must-revalidate', - cors=False -) -def swagger_ui(): - return app.swagger_ui() - - -@app.route( - '/static/{file}', - cache_control='public, max-age=86400', - cors=True -) -def static_resource(file): - return app.swagger_resource(file) +globals().update(app.default_routes()) @app.route( '/oauth2_redirect', enabled=config.google_oauth2_client_id is not None, - cache_control='no-store' + cache_control='no-store', + interactive=False, + method_spec={ + 'summary': 'Destination endpoint for Google OAuth 2.0 redirects', + 'tags': ['Auxiliary'], + 'responses': { + '200': { + 'description': fd(''' + The response body is HTML page with a script that extracts + the access token and redirects back to the Swagger UI. + ''') + } + } + } ) def oauth2_redirect(): oauth2_redirect_html = app.load_static_resource('swagger', 'oauth2-redirect.html') @@ -494,103 +493,6 @@ def oauth2_redirect(): body=oauth2_redirect_html) -common_specs = CommonEndpointSpecs(app_name='service') - - -@app.route( - '/openapi', - methods=['GET'], - cache_control='public, max-age=500', - cors=True, - **common_specs.openapi -) -def openapi(): - return Response(status_code=200, - headers={'content-type': 'application/json'}, - body=app.spec()) - - -@app.route( - '/health', - methods=['GET'], - cors=True, - **common_specs.full_health -) -def health(): - return app.health_controller.health() - - -@app.route( - '/health/basic', - methods=['GET'], - cors=True, - **common_specs.basic_health -) -def basic_health(): - return app.health_controller.basic_health() - - -@app.route( - '/health/cached', - methods=['GET'], - cors=True, - **common_specs.cached_health -) -def cached_health(): - return app.health_controller.cached_health() - - -@app.route( - '/health/fast', - methods=['GET'], - cors=True, - **common_specs.fast_health -) -def fast_health(): - return app.health_controller.fast_health() - - -@app.route( - '/health/{keys}', - methods=['GET'], - cors=True, - **common_specs.custom_health -) -def custom_health(keys: Optional[str] = None): - return app.health_controller.custom_health(keys) - - -@app.metric_alarm(metric=LambdaMetric.errors, - threshold=1, - period=24 * 60 * 60) -@app.metric_alarm(metric=LambdaMetric.throttles) -@app.retry(num_retries=0) -# FIXME: Remove redundant prefix from name -# https://github.com/DataBiosphere/azul/issues/5337 -@app.schedule( - 'rate(1 minute)', - name='servicecachehealth' -) -def update_health_cache(_event: chalice.app.CloudWatchEvent): - app.health_controller.update_cache() - - -@app.route( - '/version', - methods=['GET'], - cors=True, - **common_specs.version -) -def version(): - from azul.changelog import ( - compact_changes, - ) - return { - 'git': config.lambda_git_status, - 'changes': compact_changes(limit=10) - } - - def validate_repository_search(entity_type: EntityType, params: Mapping[str, str], **validators): @@ -835,10 +737,18 @@ def fmt_error(err_description, params): raise BRE(f'Invalid value for `{param_name}`') +deprecated_method_spec = { + 'summary': 'This endpoint will be removed in the future.', + 'tags': ['Deprecated'], + 'deprecated': True +} + + @app.route( '/integrations', methods=['GET'], - cors=True + cors=True, + method_spec=deprecated_method_spec ) def get_integrations(): query_params = app.current_request.query_params or {} @@ -2013,7 +1923,8 @@ def get_data_object_access(file_uuid, access_id): drs.dos_object_url_path('{file_uuid}'), methods=['GET'], enabled=config.is_dss_enabled(), - cors=True + cors=True, + method_spec=deprecated_method_spec ) def dos_get_data_object(file_uuid): """ diff --git a/lambdas/service/openapi.json b/lambdas/service/openapi.json index 54e1f3043d..e3e05711bb 100644 --- a/lambdas/service/openapi.json +++ b/lambdas/service/openapi.json @@ -1,9 +1,9 @@ { "openapi": "3.0.1", "info": { - "title": "azul_service", + "title": "azul-service-dev", "description": "\n# Overview\n\nAzul is a REST web service for querying metadata associated with\nboth experimental and analysis data from a data repository. In order\nto deliver response times that make it suitable for interactive use\ncases, the set of metadata properties that it exposes for sorting,\nfiltering, and aggregation is limited. Azul provides a uniform view\nof the metadata over a range of diverse schemas, effectively\nshielding clients from changes in the schemas as they occur over\ntime. It does so, however, at the expense of detail in the set of\nmetadata properties it exposes and in the accuracy with which it\naggregates them.\n\nAzul denormalizes and aggregates metadata into several different\nindices for selected entity types. Metadata entities can be queried\nusing the [Index](#operations-tag-Index) endpoints.\n\nA set of indices forms a catalog. There is a default catalog called\n`dcp2` which will be used unless a\ndifferent catalog name is specified using the `catalog` query\nparameter. Metadata from different catalogs is completely\nindependent: a response obtained by querying one catalog does not\nnecessarily correlate to a response obtained by querying another\none. Two catalogs can contain metadata from the same sources or\ndifferent sources. It is only guaranteed that the body of a\nresponse by any given endpoint adheres to one schema,\nindependently of which catalog was specified in the request.\n\nAzul provides the ability to download data and metadata via the\n[Manifests](#operations-tag-Manifests) endpoints. The\n`curl` format manifests can be used to\ndownload data files. Other formats provide various views of the\nmetadata. Manifests can be generated for a selection of files using\nfilters. These filters are interchangeable with the filters used by\nthe [Index](#operations-tag-Index) endpoints.\n\nAzul also provides a [summary](#operations-Index-get_index_summary)\nview of indexed data.\n\n## Data model\n\nAny index, when queried, returns a JSON array of hits. Each hit\nrepresents a metadata entity. Nested in each hit is a summary of the\nproperties of entities associated with the hit. An entity is\nassociated either by a direct edge in the original metadata graph,\nor indirectly as a series of edges. The nested properties are\ngrouped by the type of the associated entity. The properties of all\ndata files associated with a particular sample, for example, are\nlisted under `hits[*].files` in a `/index/samples` response. It is\nimportant to note that while each _hit_ represents a discrete\nentity, the properties nested within that hit are the result of an\naggregation over potentially many associated entities.\n\nTo illustrate this, consider a data file that is part of two\nprojects (a project is a group of related experiments, typically by\none laboratory, institution or consortium). Querying the `files`\nindex for this file yields a hit looking something like:\n\n```\n{\n \"projects\": [\n {\n \"projectTitle\": \"Project One\"\n \"laboratory\": ...,\n ...\n },\n {\n \"projectTitle\": \"Project Two\"\n \"laboratory\": ...,\n ...\n }\n ],\n \"files\": [\n {\n \"format\": \"pdf\",\n \"name\": \"Team description.pdf\",\n ...\n }\n ]\n}\n```\n\nThis example hit contains two kinds of nested entities (a hit in an\nactual Azul response will contain more): There are the two projects\nentities, and the file itself. These nested entities contain\nselected metadata properties extracted in a consistent way. This\nmakes filtering and sorting simple.\n\nAlso notice that there is only one file. When querying a particular\nindex, the corresponding entity will always be a singleton like\nthis.\n", - "version": "9.2" + "version": "9.3" }, "tags": [ { @@ -21,9 +21,29 @@ { "name": "Auxiliary", "description": "\nDescribes various aspects of the Azul service\n" + }, + { + "name": "Deprecated", + "description": "\nEndpoints that should not be used and that will be removed\n" } ], "paths": { + "/": { + "get": { + "summary": "A Swagger UI for interactive use of this REST API", + "tags": [ + "Auxiliary" + ], + "responses": { + "200": { + "description": "The response body is an HTML page containing the Swagger UI" + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" + } + } + } + }, "/openapi": { "get": { "summary": "Return OpenAPI specifications for this REST API", @@ -77,6 +97,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -84,6 +107,114 @@ ] } }, + "/static/{file}": { + "parameters": [ + { + "name": "file", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "The name of a static file to be returned" + } + ], + "get": { + "summary": "Static files needed for the Swagger UI", + "tags": [ + "Auxiliary" + ], + "responses": { + "200": { + "description": "The response body is the contents of the requested file" + }, + "404": { + "description": "The requested file does not exist" + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" + } + } + } + }, + "/version": { + "get": { + "summary": "Describe current version of this REST API", + "tags": [ + "Auxiliary" + ], + "responses": { + "200": { + "description": "Version endpoint is reachable.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "git": { + "type": "object", + "properties": { + "commit": { + "type": "string" + }, + "dirty": { + "type": "boolean" + } + }, + "required": [ + "commit", + "dirty" + ], + "additionalProperties": false + }, + "changes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "issues": { + "type": "array", + "items": { + "type": "string" + } + }, + "upgrade": { + "type": "array", + "items": { + "type": "string" + } + }, + "notes": { + "type": "string" + } + }, + "required": [ + "title", + "issues", + "upgrade" + ], + "additionalProperties": false + } + } + }, + "required": [ + "git", + "changes" + ], + "additionalProperties": false + } + } + } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" + } + } + } + }, "/health": { "get": { "summary": "Complete health check", @@ -176,6 +307,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -265,6 +399,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -358,6 +495,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -451,6 +591,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -571,6 +714,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -578,77 +724,32 @@ ] } }, - "/version": { + "/oauth2_redirect": { "get": { - "summary": "Describe current version of this REST API", + "summary": "Destination endpoint for Google OAuth 2.0 redirects", "tags": [ "Auxiliary" ], "responses": { "200": { - "description": "Version endpoint is reachable.", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "git": { - "type": "object", - "properties": { - "commit": { - "type": "string" - }, - "dirty": { - "type": "boolean" - } - }, - "required": [ - "commit", - "dirty" - ], - "additionalProperties": false - }, - "changes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "title": { - "type": "string" - }, - "issues": { - "type": "array", - "items": { - "type": "string" - } - }, - "upgrade": { - "type": "array", - "items": { - "type": "string" - } - }, - "notes": { - "type": "string" - } - }, - "required": [ - "title", - "issues", - "upgrade" - ], - "additionalProperties": false - } - } - }, - "required": [ - "git", - "changes" - ], - "additionalProperties": false - } - } - } + "description": "\nThe response body is HTML page with a script that extracts\nthe access token and redirects back to the Swagger UI.\n" + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" + } + } + } + }, + "/integrations": { + "get": { + "summary": "This endpoint will be removed in the future.", + "tags": [ + "Deprecated" + ], + "deprecated": true, + "responses": { + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -742,6 +843,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -874,6 +978,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -887,6 +994,9 @@ "responses": { "200": { "description": "\nThe HEAD method can be used to test whether an index is\noperational, or to check the validity of query parameters\nfor the [GET method](#operations-Index-get_index__entity_type_).\n" + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "parameters": [ @@ -3970,6 +4080,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } }, @@ -5573,6 +5686,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -5586,6 +5702,9 @@ "responses": { "200": { "description": "\nThe HEAD method can be used to test whether an index is\noperational, or to check the validity of query parameters\nfor the [GET method](#operations-Index-get_index_summary).\n" + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "parameters": [ @@ -6992,6 +7111,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } }, "tags": [ @@ -9705,6 +9827,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -9761,6 +9886,9 @@ }, "410": { "description": "\nThe manifest preparation job has expired. Request a\nnew preparation using the `PUT /manifest/files`\nendpoint.\n" + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -11160,6 +11288,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -11228,6 +11359,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -11363,6 +11497,9 @@ "description": "\nSet to a value that makes user agents download the file\ninstead of rendering it, suggesting a meaningful name\nfor the downloaded file stored on the user's file\nsystem. The suggested file name is taken from the\n`fileName` request parameter or, if absent, from\nmetadata describing the file. It generally does not\ncorrelate with the path component of the URL returned in\nthe `Location` header.\n" } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -11486,6 +11623,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -11546,6 +11686,9 @@ } } } + }, + "504": { + "description": "\nRequest timed out. When handling this response, clients\nshould wait the number of seconds specified in the\n`Retry-After` header and then retry the request.\n" } } } @@ -11558,7 +11701,7 @@ ], "components": { "securitySchemes": { - "azul_service": { + "azul-service-dev": { "type": "oauth2", "flows": { "implicit": { @@ -11574,7 +11717,7 @@ "security": [ {}, { - "azul_service": [ + "azul-service-dev": [ "email" ] } diff --git a/scripts/generate_openapi_document.py b/scripts/generate_openapi_document.py index 9cfae8dd22..0e9c62ef91 100644 --- a/scripts/generate_openapi_document.py +++ b/scripts/generate_openapi_document.py @@ -46,8 +46,8 @@ def main(): assert config.catalogs == catalogs with patch.object(target=config, attribute=f'{lambda_name}_function_name', - return_value=f'azul_{lambda_name}'): - assert getattr(config, f'{lambda_name}_name') == f'azul_{lambda_name}' + return_value=f'azul-{lambda_name}-dev'): + assert getattr(config, f'{lambda_name}_name') == f'azul-{lambda_name}-dev' with patch.object(target=type(config), attribute='enable_log_forwarding', new_callable=PropertyMock, diff --git a/src/azul/chalice.py b/src/azul/chalice.py index 0405133ff2..5a310df26c 100644 --- a/src/azul/chalice.py +++ b/src/azul/chalice.py @@ -18,9 +18,9 @@ from typing import ( Any, Iterator, - Optional, + Literal, Self, - Type, + Sequence, TypeVar, ) from urllib.parse import ( @@ -50,10 +50,14 @@ config, mutable_furl, open_resource, + require, ) from azul.auth import ( Authentication, ) +from azul.collections import ( + deep_dict_merge, +) from azul.enums import ( auto, ) @@ -61,6 +65,12 @@ copy_json, json_head, ) +from azul.openapi import ( + format_description, + params, + responses, + schema, +) from azul.strings import ( join_words as jw, single_quote as sq, @@ -68,7 +78,6 @@ from azul.types import ( JSON, LambdaContext, - MutableJSON, ) log = logging.getLogger(__name__) @@ -79,7 +88,7 @@ class AzulRequest(Request): Use only for type hints. The actual requests will be instances of the parent class, but they will have the attributes defined here. """ - authentication: Optional[Authentication] + authentication: Authentication | None # For some reason Chalice does not define an exception for the 410 status code @@ -121,19 +130,17 @@ class AzulChaliceApp(Chalice): def __init__(self, app_name: str, app_module_path: str, + *, unit_test: bool = False, - spec: Optional[JSON] = None): + spec: JSON): self._patch_event_source_handler() assert app_module_path.endswith('/app.py'), app_module_path self.app_module_path = app_module_path self.unit_test = unit_test self.non_interactive_routes: set[tuple[str, str]] = set() - if spec is not None: - assert 'paths' not in spec, 'The top-level spec must not define paths' - self._specs: Optional[MutableJSON] = copy_json(spec) - self._specs['paths'] = {} - else: - self._specs: Optional[MutableJSON] = None + assert 'paths' not in spec, 'The top-level spec must not define paths' + self._specs = copy_json(spec) + self._specs['paths'] = {} super().__init__(app_name, debug=config.debug > 0, configure_logs=False) # Middleware is invoked in order of registration self.register_middleware(self._logging_middleware, 'http') @@ -141,6 +148,11 @@ def __init__(self, self.register_middleware(self._api_gateway_context_middleware, 'http') self.register_middleware(self._authentication_middleware, 'http') + @property + def unqualified_app_name(self): + result, _ = config.unqualified_resource_name(self.app_name) + return result + def __call__(self, event: dict, context: LambdaContext) -> dict[str, Any]: # Chalice does not URL-decode path parameters # (https://github.com/aws/chalice/issues/511) @@ -222,23 +234,41 @@ def _security_headers_middleware(self, event, get_response): del response.headers['Content-Security-Policy'] view_function = self.routes[event.path][event.method].view_function cache_control = getattr(view_function, 'cache_control') + # Caching defeats the automatic reloading of application source code by + # `chalice local`, which is useful, so we disable caching in that case. + cache_control = 'no-store' if self.is_running_locally else cache_control response.headers['Cache-Control'] = cache_control return response + def _http_cache_for(self, seconds: int): + """ + The HTTP Cache-Control response header value that will cause the + response to the current request to be cached for the given amount of + time. + """ + return f'public, max-age={seconds}, must-revalidate' + + HttpMethod = Literal['GET', 'POST', 'PUT', 'PATCH', 'HEAD', 'OPTIONS', 'DELETE'] + + # noinspection PyMethodOverriding def route(self, path: str, + *, + methods: Sequence[HttpMethod] = ('GET',), enabled: bool = True, interactive: bool = True, cache_control: str = 'no-store', - path_spec: Optional[JSON] = None, - method_spec: Optional[JSON] = None, + path_spec: JSON | None = None, + method_spec: JSON, **kwargs): """ Decorates a view handler function in a Chalice application. See https://chalice.readthedocs.io/en/latest/api.html#Chalice.route. - :param path: See https://chalice.readthedocs.io/en/latest/api.html#Chalice.route + :param path: See https://aws.github.io/chalice/api#Chalice.route + + :param methods: See https://aws.github.io/chalice/api#Chalice.route :param enabled: If False, do not route any requests to the decorated view function. The application will behave as if the @@ -251,23 +281,23 @@ def route(self, header. :param path_spec: Corresponds to an OpenAPI Paths Object. See - https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.3.md#pathsObject + https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#paths-object If multiple `@app.route` invocations refer to the same path (but with different HTTP methods), only specify this argument for one of them, otherwise an AssertionError will be raised. :param method_spec: Corresponds to an OpenAPI Operation Object. See - https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.3.md#operationObject - This should be specified for every `@app.route` + https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#operation-object + This must be specified for every `@app.route` invocation. """ if enabled: if not interactive: - methods = kwargs['methods'] + require(bool(methods), 'Must list methods with interactive=False') self.non_interactive_routes.update((path, method) for method in methods) - methods = kwargs.get('methods', ()) - chalice_decorator = super().route(path, **kwargs) + method_spec = deep_dict_merge(method_spec, self.default_method_specs()) + chalice_decorator = super().route(path, methods=methods, **kwargs) def decorator(view_func): view_func.cache_control = cache_control @@ -323,7 +353,9 @@ def base_url(self) -> mutable_furl: callers should always append to it. """ if self.current_request is None: - # Invocation via AWS StepFunctions + # Invocation from outside the context of handling of a request, for + # example, when `chalice local` loads the app module or during an + # invocation via AWS StepFunctions self_url = config.service_endpoint elif isinstance(self.current_request, Request): try: @@ -343,10 +375,15 @@ def base_url(self) -> mutable_furl: assert False, self.current_request return self_url + @property + def is_running_locally(self) -> bool: + host = self.base_url.netloc.partition(':')[0] + return host in ('localhost', '127.0.0.1') + def _register_spec(self, path: str, - path_spec: Optional[JSON], - method_spec: Optional[JSON], + path_spec: JSON | None, + method_spec: JSON, methods: Iterable[str]): """ Add a route's specifications to the specification object. @@ -355,16 +392,15 @@ def _register_spec(self, assert path not in self._specs['paths'], 'Only specify path_spec once per route path' self._specs['paths'][path] = copy_json(path_spec) - if method_spec is not None: - for method in methods: - # OpenAPI requires HTTP method names be lower case - method = method.lower() - # This may override duplicate specs from path_specs - if path not in self._specs['paths']: - self._specs['paths'][path] = {} - assert method not in self._specs['paths'][path], \ - 'Only specify method_spec once per route path and method' - self._specs['paths'][path][method] = copy_json(method_spec) + for method in methods: + # OpenAPI requires HTTP method names be lower case + method = method.lower() + # This may override duplicate specs from path_specs + if path not in self._specs['paths']: + self._specs['paths'][path] = {} + assert method not in self._specs['paths'][path], \ + 'Only specify method_spec once per route path and method' + self._specs['paths'][path][method] = copy_json(method_spec) class _LogJSONEncoder(JSONEncoder): @@ -380,7 +416,7 @@ def default(self, o: Any) -> Any: else: return super().default(o) - def _authenticate(self) -> Optional[Authentication]: + def _authenticate(self) -> Authentication | None: """ Authenticate the current request, return None if it is unauthenticated, or raise a ChaliceViewError if it carries invalid authentication. @@ -487,7 +523,7 @@ def catalog(self) -> str: pass return config.default_catalog - def _controller(self, controller_cls: Type[C], **kwargs) -> C: + def _controller(self, controller_cls: type[C], **kwargs) -> C: return controller_cls(app=self, **kwargs) def swagger_ui(self) -> Response: @@ -563,11 +599,11 @@ class metric_alarm(HandlerDecorator): #: The number of failed or throttled lambda invocations that, when #: exceeded, will trigger the alarm. - threshold: int = attrs.field(default=0) + threshold: int #: The interval (in seconds) at which the alarm threshold is evaluated, #: ranging from 1 minute to 1 day. The default is 5 minutes. - period: int = attrs.field(default=5 * 60) + period: int def __call__(self, f): assert isinstance(f, chalice.app.EventSourceHandler), f @@ -588,13 +624,22 @@ def metric_alarms(self) -> Iterator[metric_alarm]: # The api_handler lambda functions (indexer & service) aren't # included in the app_module's handler_map, so we account for those # first. - yield self.metric_alarm(metric=metric).bind(self) + for_errors = metric is LambdaMetric.errors + alarm = self.metric_alarm(metric=metric, + threshold=1 if for_errors else 0, + period=24 * 60 * 60 if for_errors else 5 * 60) + yield alarm.bind(self) for handler_name, handler in self.handler_map.items(): if isinstance(handler, chalice.app.EventSourceHandler): try: metric_alarms = getattr(handler, 'metric_alarms') except AttributeError: - metric_alarms = (self.metric_alarm(metric=metric) for metric in LambdaMetric) + metric_alarms = ( + self.metric_alarm(metric=metric, + threshold=0, + period=5 * 60) + for metric in LambdaMetric + ) for metric_alarm in metric_alarms: yield metric_alarm.bind(self, handler_name) @@ -627,6 +672,141 @@ def retries(self) -> Iterator[retry]: else: yield retry.bind(self, handler_name) + def default_routes(self): + + @self.route( + '/', + interactive=False, + cache_control=self._http_cache_for(60), + cors=False, + method_spec={ + 'summary': 'A Swagger UI for interactive use of this REST API', + 'tags': ['Auxiliary'], + 'responses': { + '200': { + 'description': 'The response body is an HTML page containing the Swagger UI' + } + } + } + ) + def swagger_ui(): + return self.swagger_ui() + + @self.route( + '/openapi', + methods=['GET'], + cache_control=self._http_cache_for(60), + cors=True, + method_spec={ + 'summary': 'Return OpenAPI specifications for this REST API', + 'description': format_description(''' + This endpoint returns the [OpenAPI specifications]' + (https://github.com/OAI/OpenAPI-Specification) for this REST + API. These are the specifications used to generate the page + you are visiting now. + '''), + 'responses': { + '200': { + 'description': '200 response', + **responses.json_content( + schema.object( + openapi=str, + **{ + k: schema.object() + for k in ('info', 'tags', 'servers', 'paths', 'components') + } + ) + ) + } + }, + 'tags': ['Auxiliary'] + } + ) + def openapi(): + return Response(status_code=200, + headers={'content-type': 'application/json'}, + body=self.spec()) + + @self.route( + '/static/{file}', + interactive=False, + cache_control=self._http_cache_for(24 * 60 * 60), + cors=True, + method_spec={ + 'summary': 'Static files needed for the Swagger UI', + 'tags': ['Auxiliary'], + 'responses': { + '200': { + 'description': 'The response body is the contents of the requested file' + }, + '404': { + 'description': 'The requested file does not exist' + } + } + }, + path_spec={ + 'parameters': [ + params.path('file', str, description='The name of a static file to be returned') + ] + } + ) + def static_resource(file): + return self.swagger_resource(file) + + @self.route( + '/version', + methods=['GET'], + cors=True, + method_spec={ + 'summary': 'Describe current version of this REST API', + 'tags': ['Auxiliary'], + 'responses': { + '200': { + 'description': 'Version endpoint is reachable.', + **responses.json_content( + schema.object( + git=schema.object( + commit=str, + dirty=bool + ), + changes=schema.array( + schema.object( + title=str, + issues=schema.array(str), + upgrade=schema.array(str), + notes=schema.optional(str) + ) + ) + ) + ) + } + } + } + ) + def version(): + from azul.changelog import ( + compact_changes, + ) + return { + 'git': config.lambda_git_status, + 'changes': compact_changes(limit=10) + } + + return locals() + + def default_method_specs(self): + return { + 'responses': { + '504': { + 'description': format_description(''' + Request timed out. When handling this response, clients + should wait the number of seconds specified in the + `Retry-After` header and then retry the request. + ''') + } + } + } + @attrs.frozen(kw_only=True) class AppController: diff --git a/src/azul/health.py b/src/azul/health.py index 128c279226..c8835befcb 100644 --- a/src/azul/health.py +++ b/src/azul/health.py @@ -19,6 +19,7 @@ from botocore.exceptions import ( ClientError, ) +import chalice from chalice import ( ChaliceViewError, NotFoundError, @@ -40,6 +41,8 @@ ) from azul.chalice import ( AppController, + AzulChaliceApp, + LambdaMetric, ) from azul.deployment import ( aws, @@ -47,6 +50,12 @@ from azul.es import ( ESClientFactory, ) +from azul.openapi import ( + format_description, + params, + responses, + schema, +) from azul.plugins import ( MetadataPlugin, ) @@ -314,3 +323,204 @@ def as_json_fast(self) -> JSON: ) all_keys: ClassVar[Set[str]] = frozenset(p.key for p in all_properties) + + +class HealthApp(AzulChaliceApp): + + @cached_property + def health_controller(self) -> HealthController: + return self._controller(HealthController, + lambda_name=self.unqualified_app_name) + + def default_routes(self): + _routes = super().default_routes() + _app_name = self.unqualified_app_name + + _up_key = { + 'up': format_description(''' + indicates the overall result of the health check + '''), + } + + _fast_keys = { + **{ + prop.key: format_description(prop.description) + for prop in Health.fast_properties[_app_name] + }, + **_up_key + } + + _all_keys = { + **{ + prop.key: format_description(prop.description) + for prop in Health.all_properties + }, + **_up_key + } + + def _health_spec(health_keys: dict) -> JSON: + return { + 'responses': { + f'{200 if up else 503}': { + 'description': format_description(f''' + {'The' if up else 'At least one of the'} checked resources + {'are' if up else 'is not'} healthy. + + The response consists of the following keys: + + ''') + ''.join(f'* `{k}` {v}' for k, v in health_keys.items()) + format_description(f''' + + The top-level `up` key of the response is + `{'true' if up else 'false'}`. + + ''') + (format_description(f''' + {'All' if up else 'At least one'} of the nested `up` keys + {'are `true`' if up else 'is `false`'}. + ''') if len(health_keys) > 1 else ''), + **responses.json_content( + schema.object( + additional_properties=schema.object( + additional_properties=True, + up=schema.enum(up) + ), + up=schema.enum(up) + ), + example={ + k: up if k == 'up' else {} for k in health_keys + } + ) + } for up in [True, False] + }, + 'tags': ['Auxiliary'] + } + + @self.route( + '/health', + methods=['GET'], + cors=True, + method_spec={ + 'summary': 'Complete health check', + 'description': format_description(f''' + Health check of the {_app_name} REST API and all + resources it depends on. This may take long time to complete + and exerts considerable load on the API. For that reason it + should not be requested frequently or by automated + monitoring facilities that would be better served by the + [`/health/fast`](#operations-Auxiliary-get_health_fast) or + [`/health/cached`](#operations-Auxiliary-get_health_cached) + endpoints. + '''), + **_health_spec(_all_keys) + } + ) + def health(): + return self.health_controller.health() + + @self.route( + '/health/basic', + methods=['GET'], + cors=True, + method_spec={ + 'summary': 'Basic health check', + 'description': format_description(f''' + Health check of only the REST API itself, excluding other + resources that it depends on. A 200 response indicates that + the {_app_name} is reachable via HTTP(S) but nothing + more. + '''), + **_health_spec(_up_key) + } + ) + def basic_health(): + return self.health_controller.basic_health() + + @self.route( + '/health/cached', + methods=['GET'], + cors=True, + method_spec={ + 'summary': 'Cached health check for continuous monitoring', + 'description': format_description(f''' + Return a cached copy of the + [`/health/fast`](#operations-Auxiliary-get_health_fast) + response. This endpoint is optimized for continuously + running, distributed health monitors such as Route 53 health + checks. The cache ensures that the {_app_name} is not + overloaded by these types of health monitors. The cache is + updated every minute. + '''), + **_health_spec(_fast_keys) + } + ) + def cached_health(): + return self.health_controller.cached_health() + + @self.route( + '/health/fast', + methods=['GET'], + cors=True, + method_spec={ + 'summary': 'Fast health check', + 'description': format_description(''' + Performance-optimized health check of the REST API and other + critical resources tht it depends on. This endpoint can be + requested more frequently than + [`/health`](#operations-Auxiliary-get_health) but + periodically scheduled, automated requests should be made to + [`/health/cached`](#operations-Auxiliary-get_health_cached). + '''), + **_health_spec(_fast_keys) + } + ) + def fast_health(): + return self.health_controller.fast_health() + + @self.route( + '/health/{keys}', + methods=['GET'], + cors=True, + method_spec={ + 'summary': 'Selective health check', + 'description': format_description(''' + This endpoint allows clients to request a health check on a + specific set of resources. Each resource is identified by a + *key*, the same key under which the resource appears in a + [`/health`](#operations-Auxiliary-get_health) response. + '''), + **_health_spec(_all_keys) + }, path_spec={ + 'parameters': [ + params.path( + 'keys', + type_=schema.array(schema.enum(*sorted(Health.all_keys))), + description=''' + A comma-separated list of keys selecting the health + checks to be performed. Each key corresponds to an + entry in the response. + ''') + ] + } + ) + def custom_health(keys: Optional[str] = None): + return self.health_controller.custom_health(keys) + + @self.metric_alarm(metric=LambdaMetric.errors, + threshold=1, + period=24 * 60 * 60) + @self.metric_alarm(metric=LambdaMetric.throttles, + threshold=0, + period=5 * 60) + @self.retry(num_retries=0) + # FIXME: Remove redundant prefix from name + # https://github.com/DataBiosphere/azul/issues/5337 + @self.schedule( + 'rate(1 minute)', + name=self.unqualified_app_name + 'cachehealth' + ) + def update_health_cache(_event: chalice.app.CloudWatchEvent): + self.health_controller.update_cache() + + return { + **_routes, + **{k: v for k, v in locals().items() if not k.startswith('_')} + } diff --git a/src/azul/openapi/spec.py b/src/azul/openapi/spec.py deleted file mode 100644 index c14708aa36..0000000000 --- a/src/azul/openapi/spec.py +++ /dev/null @@ -1,234 +0,0 @@ -import attr - -from azul import ( - JSON, -) -from azul.health import ( - Health, -) -from azul.openapi import ( - format_description, - params, - responses, - schema, -) - - -@attr.s(auto_attribs=True, frozen=True, kw_only=True) -class CommonEndpointSpecs: - app_name: str - - _up_key = { - 'up': format_description(''' - indicates the overall result of the health check - '''), - } - - @property - def _fast_keys(self): - return { - **{ - prop.key: format_description(prop.description) - for prop in Health.fast_properties[self.app_name] - }, - **self._up_key - } - - _all_keys = { - **{ - prop.key: format_description(prop.description) - for prop in Health.all_properties - }, - **_up_key - } - - def _health_spec(self, health_keys: dict) -> JSON: - return { - 'responses': { - f'{200 if up else 503}': { - 'description': format_description(f''' - {'The' if up else 'At least one of the'} checked resources - {'are' if up else 'is not'} healthy. - - The response consists of the following keys: - - ''') + ''.join(f'* `{k}` {v}' for k, v in health_keys.items()) + format_description(f''' - - The top-level `up` key of the response is - `{'true' if up else 'false'}`. - - ''') + (format_description(f''' - {'All' if up else 'At least one'} of the nested `up` keys - {'are `true`' if up else 'is `false`'}. - ''') if len(health_keys) > 1 else ''), - **responses.json_content( - schema.object( - additional_properties=schema.object( - additional_properties=True, - up=schema.enum(up) - ), - up=schema.enum(up) - ), - example={ - k: up if k == 'up' else {} for k in health_keys - } - ) - } for up in [True, False] - }, - 'tags': ['Auxiliary'] - } - - @property - def full_health(self) -> JSON: - return { - 'method_spec': { - 'summary': 'Complete health check', - 'description': format_description(f''' - Health check of the {self.app_name} REST API and all - resources it depends on. This may take long time to complete - and exerts considerable load on the API. For that reason it - should not be requested frequently or by automated - monitoring facilities that would be better served by the - [`/health/fast`](#operations-Auxiliary-get_health_fast) or - [`/health/cached`](#operations-Auxiliary-get_health_cached) - endpoints. - '''), - **self._health_spec(self._all_keys) - } - } - - @property - def basic_health(self) -> JSON: - return { - 'method_spec': { - 'summary': 'Basic health check', - 'description': format_description(f''' - Health check of only the REST API itself, excluding other - resources that it depends on. A 200 response indicates that - the {self.app_name} is reachable via HTTP(S) but nothing - more. - '''), - **self._health_spec(self._up_key) - } - } - - @property - def cached_health(self) -> JSON: - return { - 'method_spec': { - 'summary': 'Cached health check for continuous monitoring', - 'description': format_description(f''' - Return a cached copy of the - [`/health/fast`](#operations-Auxiliary-get_health_fast) - response. This endpoint is optimized for continuously - running, distributed health monitors such as Route 53 health - checks. The cache ensures that the {self.app_name} is not - overloaded by these types of health monitors. The cache is - updated every minute. - '''), - **self._health_spec(self._fast_keys) - } - } - - @property - def fast_health(self) -> JSON: - return { - 'method_spec': { - 'summary': 'Fast health check', - 'description': format_description(''' - Performance-optimized health check of the REST API and other - critical resources tht it depends on. This endpoint can be - requested more frequently than - [`/health`](#operations-Auxiliary-get_health) but - periodically scheduled, automated requests should be made to - [`/health/cached`](#operations-Auxiliary-get_health_cached). - '''), - **self._health_spec(self._fast_keys) - } - } - - @property - def custom_health(self) -> JSON: - return { - 'method_spec': { - 'summary': 'Selective health check', - 'description': format_description(''' - This endpoint allows clients to request a health check on a - specific set of resources. Each resource is identified by a - *key*, the same key under which the resource appears in a - [`/health`](#operations-Auxiliary-get_health) response. - '''), - **self._health_spec(self._all_keys) - }, - 'path_spec': { - 'parameters': [ - params.path( - 'keys', - type_=schema.array(schema.enum(*sorted(Health.all_keys))), - description=''' - A comma-separated list of keys selecting the health - checks to be performed. Each key corresponds to an - entry in the response. - ''') - ] - } - } - - @property - def openapi(self) -> JSON: - return { - 'method_spec': { - 'summary': 'Return OpenAPI specifications for this REST API', - 'description': format_description(''' - This endpoint returns the [OpenAPI specifications]' - (https://github.com/OAI/OpenAPI-Specification) for this REST - API. These are the specifications used to generate the page - you are visiting now. - '''), - 'responses': { - '200': { - 'description': '200 response', - **responses.json_content( - schema.object( - openapi=str, - **{ - k: schema.object() - for k in ('info', 'tags', 'servers', 'paths', 'components') - } - ) - ) - } - }, - 'tags': ['Auxiliary'] - } - } - - @property - def version(self) -> JSON: - return { - 'method_spec': { - 'summary': 'Describe current version of this REST API', - 'tags': ['Auxiliary'], - 'responses': { - '200': { - 'description': 'Version endpoint is reachable.', - **responses.json_content( - schema.object( - git=schema.object( - commit=str, - dirty=bool - ), - changes=schema.array( - schema.object( - title=str, - issues=schema.array(str), - upgrade=schema.array(str), - notes=schema.optional(str) - ) - ) - ) - ) - } - } - } - } diff --git a/src/azul/terraform.py b/src/azul/terraform.py index f40382e80f..4166c885a6 100644 --- a/src/azul/terraform.py +++ b/src/azul/terraform.py @@ -814,25 +814,40 @@ def tf_config(self, app_name): assert 'minimum_compression_size' not in rest_api, rest_api key = 'x-amazon-apigateway-minimum-compression-size' openapi_spec[key] = config.minimum_compression_size - assert 'aws_api_gateway_gateway_response' not in resources, resources - openapi_spec['x-amazon-apigateway-gateway-responses'] = { - f'DEFAULT_{response_type}': { - 'responseParameters': { - # Static value response header parameters must be enclosed - # within a pair of single quotes. - # - # https://docs.aws.amazon.com/apigateway/latest/developerguide/request-response-data-mappings.html#mapping-response-parameters - # https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-swagger-extensions-gateway-responses.html - # - # Note that azul.strings.single_quote() is not used here - # since API Gateway allows internal single quotes in the - # value, which that function would prohibit. - # - f'gatewayresponse.header.{k}': f"'{v}'" - for k, v in AzulChaliceApp.security_headers.items() - } - } for response_type in ['4XX', '5XX'] + + # When mapping a static value to a response parameter, the value + # must be enclosed within a pair of single quotes. Note that + # azul.strings.single_quote() is not used here since API Gateway allows + # internal single quotes, which that function would prohibit. + # + # https://docs.aws.amazon.com/apigateway/latest/developerguide/request-response-data-mappings.html#mapping-response-parameters + # + security_headers = { + f'gatewayresponse.header.{k}': f"'{v}'" + for k, v in AzulChaliceApp.security_headers.items() } + assert 'aws_api_gateway_gateway_response' not in resources, resources + openapi_spec['x-amazon-apigateway-gateway-responses'] = ( + { + f'DEFAULT_{response_type}': { + 'responseParameters': security_headers + } for response_type in ['4XX', '5XX'] + } | { + response_type: { + 'responseParameters': { + **security_headers, + 'gatewayresponse.header.Retry-After': "'10'" + }, + 'responseTemplates': { + "application/json": json.dumps({ + 'message': '504 Gateway Timeout. Wait the number of ' + 'seconds specified in the `Retry-After` ' + 'header before retrying the request.' + }) + } + } for response_type in ['INTEGRATION_TIMEOUT', 'INTEGRATION_FAILURE'] + } + ) locals[app_name] = json.dumps(openapi_spec) return { diff --git a/test/integration_test.py b/test/integration_test.py index c9a79f0a4e..4298eb2abb 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -1990,15 +1990,19 @@ def test(self): class ResponseHeadersTest(AzulTestCase): def test_response_security_headers(self): + no_cache = 'no-store' + short_cache = 'public, max-age=60, must-revalidate' + long_cache = 'public, max-age=86400, must-revalidate' test_cases = { - '/': {'Cache-Control': 'public, max-age=0, must-revalidate'}, - '/static/swagger-ui.css': {'Cache-Control': 'public, max-age=86400'}, - '/openapi': {'Cache-Control': 'public, max-age=500'}, - '/oauth2_redirect': {'Cache-Control': 'no-store'}, - '/health/basic': {'Cache-Control': 'no-store'} + '/': short_cache, + '/static/swagger-ui.css': long_cache, + '/openapi': short_cache, + '/oauth2_redirect': no_cache, + '/health/basic': no_cache } for endpoint in (config.service_endpoint, config.indexer_endpoint): - for path, expected_headers in test_cases.items(): + for path, cache_control in test_cases.items(): + expected_headers = {'Cache-Control': cache_control} with self.subTest(endpoint=endpoint, path=path): if path == '/oauth2_redirect' and endpoint == config.indexer_endpoint: pass # no oauth2 endpoint on indexer Lambda diff --git a/test/test_app_logging.py b/test/test_app_logging.py index 2372707577..a9c41957db 100644 --- a/test/test_app_logging.py +++ b/test/test_app_logging.py @@ -55,10 +55,10 @@ def test(self): with mock.patch.dict(os.environ, AZUL_DEBUG=str(debug)): with self.subTest(debug=debug): log_level = azul_log_level() - app = AzulChaliceApp(__name__, '/app.py', unit_test=True) + app = AzulChaliceApp(__name__, '/app.py', unit_test=True, spec={}) path = '/fail/path' - @app.route(path) + @app.route(path, method_spec={}) def fail(): raise ValueError(magic_message) diff --git a/test/test_openapi.py b/test/test_openapi.py index 0fb24e034c..e37643819f 100644 --- a/test/test_openapi.py +++ b/test/test_openapi.py @@ -6,9 +6,15 @@ furl, ) +from azul import ( + JSON, +) from azul.chalice import ( AzulChaliceApp, ) +from azul.json import ( + copy_json, +) from azul.logging import ( configure_test_logging, ) @@ -35,9 +41,11 @@ def app(self, spec): def test_top_level_spec(self): spec = {'foo': 'bar'} app = self.app(spec) - self.assertEqual(app._specs, {'foo': 'bar', 'paths': {}}, "Confirm 'paths' is added") + self.assertEqual({'foo': 'bar', 'paths': {}}, app._specs, + "Confirm 'paths' is added") spec['new key'] = 'new value' - self.assertNotIn('new key', app.spec(), 'Changing input object should not affect specs') + self.assertNotIn('new key', app.spec(), + 'Changing input object should not affect specs') def test_already_annotated_top_level_spec(self): with self.assertRaises(AssertionError): @@ -46,17 +54,18 @@ def test_already_annotated_top_level_spec(self): def test_unannotated(self): app = self.app({'foo': 'bar'}) - @app.route('/foo', methods=['GET', 'PUT']) + @app.route('/foo', methods=['GET', 'PUT'], method_spec={}) def route(): pass # no coverage expected = { 'foo': 'bar', - 'paths': {}, + 'paths': {'/foo': {'get': {}, 'put': {}}}, 'tags': [], 'servers': [{'url': 'https://fake.url/'}] } - self.assertEqual(app.spec(), expected) + actual_spec = self._assert_default_method_spec(app.spec()) + self.assertEqual(expected, actual_spec) def test_just_method_spec(self): app = self.app({'foo': 'bar'}) @@ -76,24 +85,22 @@ def route(): 'tags': [], 'servers': [{'url': 'https://fake.url/'}] } - self.assertEqual(app.spec(), expected_spec) - - def test_just_path_spec(self): - app = self.app({'foo': 'bar'}) - - @app.route('/foo', methods=['GET', 'PUT'], path_spec={'a': 'b'}) - def route(): - pass # no coverage - expected_spec = { - 'foo': 'bar', - 'paths': { - '/foo': {'a': 'b'} - }, - 'tags': [], - 'servers': [{'url': 'https://fake.url/'}] - } - self.assertEqual(app.spec(), expected_spec) + actual_spec = self._assert_default_method_spec(app.spec()) + self.assertEqual(expected_spec, actual_spec) + + def _assert_default_method_spec(self, actual_spec: JSON) -> JSON: + actual_spec = copy_json(actual_spec) + for path_spec in actual_spec['paths'].values(): + for method, method_spec in path_spec.items(): + methods = {'get', 'put'} # only what's used in these tests + if method in methods: + responses = method_spec.pop('responses') + response = responses.pop('504') + description = response.pop('description') + self.assertIn('Request timed out', description) + self.assertEqual(({}, {}), (response, responses)) + return actual_spec def test_fully_annotated_override(self): app = self.app({'foo': 'bar'}) @@ -132,7 +139,8 @@ def route(): 'tags': [], 'servers': [{'url': 'https://fake.url/'}] } - self.assertEqual(app.spec(), expected_specs) + actual_spec = self._assert_default_method_spec(app.spec()) + self.assertEqual(expected_specs, actual_spec) def test_duplicate_method_specs(self): app = self.app({'foo': 'bar'}) @@ -147,12 +155,12 @@ def route(): def test_duplicate_path_specs(self): app = self.app({'foo': 'bar'}) - @app.route('/foo', methods=['PUT'], path_spec={'a': 'XXX'}) + @app.route('/foo', methods=['PUT'], path_spec={'a': 'XXX'}, method_spec={}) def route1(): pass with self.assertRaises(AssertionError) as cm: - @app.route('/foo', methods=['GET'], path_spec={'a': 'b'}) + @app.route('/foo', methods=['GET'], path_spec={'a': 'b'}, method_spec={}) def route2(): pass self.assertEqual(str(cm.exception), 'Only specify path_spec once per route path')