diff --git a/.gitignore b/.gitignore index 97188980..5750a8d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Python *.pyc /venv +.venv # IDE .idea diff --git a/Makefile.config.example b/Makefile.config.example index 6c5f0d4e..593f5eef 100644 --- a/Makefile.config.example +++ b/Makefile.config.example @@ -46,6 +46,7 @@ LOG_TYPE := flat # For NGAP Deployments # ######################## # In an NGAP environment you must uncomment the following line or the deployment will fail +# This has been deprecated in NGAP and is no longer necessary # PERMISSION_BOUNDARY_NAME := NGAPShRoleBoundary PRIVATE_VPC := diff --git a/README.MD b/README.MD index b0b2bf0e..0211fa7b 100644 --- a/README.MD +++ b/README.MD @@ -143,6 +143,7 @@ container path: file:/var/deps/rain-api-core ``` *NOTE: This is a generated file and you should NOT be committing these changes* + 3. Add any source files of that dependency to the `REQUIREMENTS_DEPS` in your `Makefile.config`: ```makefile REQUIREMENTS_DEPS := $(shell find /host/path/to/rain-api-core/rain_api_core/ -name '*.py') diff --git a/requirements/requirements.in b/requirements/requirements.in index 75e6933c..7026c0af 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -1,5 +1,5 @@ cachetools cfnresponse chalice -git+https://github.com/asfadmin/rain-api-core.git@318aac226c92cf6f60cc8821d6d94669485972c6 +git+https://github.com/asfadmin/rain-api-core.git@a9a00d126878f56213af972f6fb5bf6bb1490909 netaddr diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 68a46897..dc10f91b 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -50,7 +50,7 @@ pyyaml==6.0.1 # via # chalice # rain-api-core -rain-api-core @ git+https://github.com/asfadmin/rain-api-core.git@318aac226c92cf6f60cc8821d6d94669485972c6 +rain-api-core @ git+https://github.com/asfadmin/rain-api-core.git@a9a00d126878f56213af972f6fb5bf6bb1490909 # via -r requirements/requirements.in readchar==4.0.5 # via inquirer diff --git a/tests/test_app.py b/tests/test_app.py index d5d2fd8d..f79b630b 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -589,7 +589,7 @@ def test_try_download_from_bucket( client.get_bucket_location.return_value = {"LocationConstraint": "us-east-1"} client.head_object.return_value = {"ContentLength": 2048} - response = app.try_download_from_bucket("somebucket", "somefile", user_profile, {}) + response = app.try_download_from_bucket("somebucket", "somefile", user_profile, {}, api_request_uuid=None) client.head_object.assert_called_once() assert response.body == "" assert response.status_code == 303 @@ -606,7 +606,7 @@ def test_try_download_from_bucket( monkeypatch.setenv("AWS_DEFAULT_REGION", "us-west-2") client.head_object.reset_mock() - response = app.try_download_from_bucket("somebucket", "somefile", user_profile, "not a dict") + response = app.try_download_from_bucket("somebucket", "somefile", user_profile, "not a dict", api_request_uuid=None) client.head_object.assert_not_called() assert response.body == "" assert response.status_code == 303 @@ -635,7 +635,7 @@ def test_try_download_from_bucket_client_error( mock_get_role_creds.return_value = (mock.Mock(), 1000) mock_get_role_session().client.side_effect = ClientError({}, "bar") - app.try_download_from_bucket("somebucket", "somefile", user_profile, {}) + app.try_download_from_bucket("somebucket", "somefile", user_profile, {}, None) mock_make_html_response.assert_called_once_with( { "contentstring": "There was a problem accessing download data.", @@ -672,7 +672,7 @@ def test_try_download_from_bucket_not_found( "bar" ) - app.try_download_from_bucket("somebucket", "somefile", user_profile, {}) + app.try_download_from_bucket("somebucket", "somefile", user_profile, {}, None) mock_make_html_response.assert_called_once_with( { "contentstring": "Could not find requested data.", @@ -710,7 +710,7 @@ def test_try_download_from_bucket_invalid_range( "bar" ) - response = app.try_download_from_bucket("somebucket", "somefile", user_profile, {}) + response = app.try_download_from_bucket("somebucket", "somefile", user_profile, {}, None) assert response.body == "Invalid Range" assert response.status_code == 416 assert response.headers == {} @@ -1197,6 +1197,7 @@ def test_dynamic_url_head_missing_proxy(mock_get_yaml_file, current_request): @mock.patch(f"{MODULE}.get_yaml_file", autospec=True) +@mock.patch(f"{MODULE}.get_api_request_uuid", autospec=True) @mock.patch(f"{MODULE}.try_download_from_bucket", autospec=True) @mock.patch(f"{MODULE}.JwtManager.get_profile_from_headers", autospec=True) @mock.patch(f"{MODULE}.JWT_COOKIE_NAME", "asf-cookie") @@ -1204,6 +1205,7 @@ def test_dynamic_url_head_missing_proxy(mock_get_yaml_file, current_request): def test_dynamic_url( mock_get_profile, mock_try_download_from_bucket, + mock_get_api_request_uuid, mock_get_yaml_file, data_path, user_profile, @@ -1215,6 +1217,7 @@ def test_dynamic_url( mock_get_yaml_file.return_value = yaml.full_load(f) mock_get_profile.return_value = user_profile + mock_get_api_request_uuid.return_value = None current_request.uri_params = {"proxy": "DATA-TYPE-1/PLATFORM-A/OBJECT_1"} # Can't use the chalice test client here as it doesn't seem to understand the `{proxy+}` route @@ -1224,12 +1227,14 @@ def test_dynamic_url( "gsfc-ngap-d-pa-dt1", "OBJECT_1", user_profile, - {} + {}, + None ) assert response is MOCK_RESPONSE @mock.patch(f"{MODULE}.get_yaml_file", autospec=True) +@mock.patch(f"{MODULE}.get_api_request_uuid", autospec=True) @mock.patch(f"{MODULE}.try_download_from_bucket", autospec=True) @mock.patch(f"{MODULE}.JwtManager.get_profile_from_headers", autospec=True) @mock.patch(f"{MODULE}.JWT_COOKIE_NAME", "asf-cookie") @@ -1237,6 +1242,7 @@ def test_dynamic_url( def test_dynamic_url_public_unauthenticated( mock_get_profile, mock_try_download_from_bucket, + mock_get_api_request_uuid, mock_get_yaml_file, data_path, current_request @@ -1247,16 +1253,18 @@ def test_dynamic_url_public_unauthenticated( mock_get_yaml_file.return_value = yaml.full_load(f) mock_get_profile.return_value = None + mock_get_api_request_uuid.return_value = None current_request.uri_params = {"proxy": "BROWSE/PLATFORM-A/OBJECT_2"} # Can't use the chalice test client here as it doesn't seem to understand the `{proxy+}` route response = app.dynamic_url() - mock_try_download_from_bucket.assert_called_once_with("gsfc-ngap-d-pa-bro", "OBJECT_2", None, {}) + mock_try_download_from_bucket.assert_called_once_with("gsfc-ngap-d-pa-bro", "OBJECT_2", None, {}, None) assert response is MOCK_RESPONSE @mock.patch(f"{MODULE}.get_yaml_file", autospec=True) +@mock.patch(f"{MODULE}.get_api_request_uuid", autospec=True) @mock.patch(f"{MODULE}.try_download_from_bucket", autospec=True) @mock.patch(f"{MODULE}.JwtManager.get_profile_from_headers", autospec=True) @mock.patch(f"{MODULE}.JWT_COOKIE_NAME", "asf-cookie") @@ -1264,6 +1272,7 @@ def test_dynamic_url_public_unauthenticated( def test_dynamic_url_public_authenticated( mock_get_profile, mock_try_download_from_bucket, + mock_get_api_request_uuid, mock_get_yaml_file, data_path, user_profile, @@ -1275,16 +1284,18 @@ def test_dynamic_url_public_authenticated( mock_get_yaml_file.return_value = yaml.full_load(f) mock_get_profile.return_value = user_profile + mock_get_api_request_uuid.return_value = None current_request.uri_params = {"proxy": "BROWSE/PLATFORM-A/OBJECT_2"} # Can't use the chalice test client here as it doesn't seem to understand the `{proxy+}` route response = app.dynamic_url() - mock_try_download_from_bucket.assert_called_once_with("gsfc-ngap-d-pa-bro", "OBJECT_2", user_profile, {}) + mock_try_download_from_bucket.assert_called_once_with("gsfc-ngap-d-pa-bro", "OBJECT_2", user_profile, {}, None) assert response is MOCK_RESPONSE @mock.patch(f"{MODULE}.get_yaml_file", autospec=True) +@mock.patch(f"{MODULE}.get_api_request_uuid", autospec=True) @mock.patch(f"{MODULE}.try_download_from_bucket", autospec=True) @mock.patch(f"{MODULE}.JwtManager.get_profile_from_headers", autospec=True) @mock.patch(f"{MODULE}.JWT_COOKIE_NAME", "asf-cookie") @@ -1292,6 +1303,7 @@ def test_dynamic_url_public_authenticated( def test_dynamic_url_public_custom_headers( mock_get_profile, mock_try_download_from_bucket, + mock_get_api_request_uuid, mock_get_yaml_file, data_path, current_request @@ -1302,6 +1314,7 @@ def test_dynamic_url_public_custom_headers( mock_get_yaml_file.return_value = yaml.full_load(f) mock_get_profile.return_value = None + mock_get_api_request_uuid.return_value = None current_request.uri_params = {"proxy": "HEADERS/BROWSE/OBJECT_1"} # Can't use the chalice test client here as it doesn't seem to understand the `{proxy+}` route @@ -1314,12 +1327,14 @@ def test_dynamic_url_public_custom_headers( { "custom-header-1": "custom-header-1-value", "custom-header-2": "custom-header-2-value" - } + }, + None ) assert response is MOCK_RESPONSE @mock.patch(f"{MODULE}.get_yaml_file", autospec=True) +@mock.patch(f"{MODULE}.get_api_request_uuid", autospec=True) @mock.patch(f"{MODULE}.try_download_from_bucket", autospec=True) @mock.patch(f"{MODULE}.user_in_group", autospec=True) @mock.patch(f"{MODULE}.JwtManager.get_profile_from_headers", autospec=True) @@ -1331,6 +1346,7 @@ def test_dynamic_url_private( mock_get_profile, mock_user_in_group, mock_try_download_from_bucket, + mock_get_api_request_uuid, mock_get_yaml_file, data_path, user_profile, @@ -1344,6 +1360,7 @@ def test_dynamic_url_private( mock_get_yaml_file.return_value = yaml.full_load(f) mock_get_profile.return_value = user_profile + mock_get_api_request_uuid.return_value = None current_request.uri_params = {"proxy": "PRIVATE/PLATFORM-A/OBJECT_2"} # Can't use the chalice test client here as it doesn't seem to understand the `{proxy+}` route @@ -1353,12 +1370,14 @@ def test_dynamic_url_private( "gsfc-ngap-d-pa-priv", "OBJECT_2", user_profile, - {"SET-COOKIE": "cookie"} + {"SET-COOKIE": "cookie"}, + None ) assert response is MOCK_RESPONSE @mock.patch(f"{MODULE}.get_yaml_file", autospec=True) +@mock.patch(f"{MODULE}.get_api_request_uuid", autospec=True) @mock.patch(f"{MODULE}.try_download_from_bucket", autospec=True) @mock.patch(f"{MODULE}.user_in_group", autospec=True) @mock.patch(f"{MODULE}.JwtManager.get_profile_from_headers", autospec=True) @@ -1370,6 +1389,7 @@ def test_dynamic_url_private_custom_headers( mock_get_profile, mock_user_in_group, mock_try_download_from_bucket, + mock_get_api_request_uuid, mock_get_yaml_file, data_path, user_profile, @@ -1384,6 +1404,7 @@ def test_dynamic_url_private_custom_headers( mock_get_yaml_file.return_value = yaml.full_load(f) mock_get_profile.return_value = user_profile + mock_get_api_request_uuid.return_value = None current_request.uri_params = {"proxy": "HEADERS/PRIVATE/OBJECT_1"} # Can't use the chalice test client here as it doesn't seem to understand the `{proxy+}` route @@ -1397,12 +1418,14 @@ def test_dynamic_url_private_custom_headers( "custom-header-3": "custom-header-3-value", "custom-header-4": "custom-header-4-value", "SET-COOKIE": "cookie" - } + }, + None ) assert response is MOCK_RESPONSE @mock.patch(f"{MODULE}.get_yaml_file", autospec=True) +@mock.patch(f"{MODULE}.get_api_request_uuid", autospec=True) @mock.patch(f"{MODULE}.try_download_from_bucket", autospec=True) @mock.patch(f"{MODULE}.JwtManager.get_profile_from_headers", autospec=True) @mock.patch(f"{MODULE}.JWT_COOKIE_NAME", "asf-cookie") @@ -1410,6 +1433,7 @@ def test_dynamic_url_private_custom_headers( def test_dynamic_url_public_within_private( mock_get_profile_from_headers, mock_try_download_from_bucket, + mock_get_api_request_uuid, mock_get_yaml_file, current_request ): @@ -1427,12 +1451,13 @@ def test_dynamic_url_public_within_private( } mock_get_profile_from_headers.return_value = None + mock_get_api_request_uuid.return_value = None current_request.uri_params = {"proxy": "FOO/BROWSE/OBJECT_1"} # Can't use the chalice test client here as it doesn't seem to understand the `{proxy+}` route response = app.dynamic_url() - mock_try_download_from_bucket.assert_called_once_with("gsfc-ngap-d-bucket", "BROWSE/OBJECT_1", None, {}) + mock_try_download_from_bucket.assert_called_once_with("gsfc-ngap-d-bucket", "BROWSE/OBJECT_1", None, {}, None) assert response is MOCK_RESPONSE @@ -1503,6 +1528,7 @@ def test_dynamic_url_directory( @mock.patch(f"{MODULE}.get_yaml_file", autospec=True) +@mock.patch(f"{MODULE}.get_api_request_uuid", autospec=True) @mock.patch(f"{MODULE}.try_download_from_bucket", autospec=True) @mock.patch(f"{MODULE}.JwtManager.get_profile_from_headers", autospec=True) @mock.patch(f"{MODULE}.RequestAuthorizer._handle_auth_bearer_header", autospec=True) @@ -1514,6 +1540,7 @@ def test_dynamic_url_bearer_auth( mock_handle_auth_bearer_header, mock_get_profile, mock_try_download_from_bucket, + mock_get_api_request_uuid, mock_get_yaml_file, data_path, user_profile, @@ -1526,6 +1553,7 @@ def test_dynamic_url_bearer_auth( mock_get_yaml_file.return_value = yaml.full_load(f) mock_get_profile.return_value = None + mock_get_api_request_uuid.return_value = None current_request.uri_params = {"proxy": "DATA-TYPE-1/PLATFORM-A/OBJECT_1"} current_request.headers = {"Authorization": "bearer b64token"} @@ -1536,7 +1564,8 @@ def test_dynamic_url_bearer_auth( "gsfc-ngap-d-pa-dt1", "OBJECT_1", user_profile, - {"SET-COOKIE": "cookie"} + {"SET-COOKIE": "cookie"}, + None ) assert response.body == "Mock response" assert response.status_code == 200 @@ -1687,3 +1716,14 @@ def test_x_origin_request_id_forwarded(mock_retrieve_secret, client): response = client.http.get("/profile", headers={"x-origin-request-id": "x_origin_request_1234"}) assert response.headers["x-origin-request-id"] == "x_origin_request_1234" + + +def test_get_api_request_uuid(): + response = app.get_api_request_uuid({"A-api-request-uuid": "test-uuid"}) + assert response == "test-uuid" + + response = app.get_api_request_uuid({"A-userid": "test-userid"}) + assert response is None + + response = app.get_api_request_uuid(None) + assert response is None diff --git a/thin_egress_app/app.py b/thin_egress_app/app.py index 9a7ede49..fd9bf7c1 100644 --- a/thin_egress_app/app.py +++ b/thin_egress_app/app.py @@ -387,6 +387,15 @@ def get_user_from_token(token): return None +@with_trace() +def get_api_request_uuid(query_params): + if query_params is not None: + api_request_uuid = query_params.get("A-api-request-uuid") + if api_request_uuid is not None: + log.info("A-api-request-uuid query param is " + api_request_uuid) + return api_request_uuid + + @with_trace() def cumulus_log_message(outcome: str, code: int, http_method: str, k_v: dict): k_v.update({ @@ -583,7 +592,7 @@ def get_user_ip(): @with_trace() -def try_download_from_bucket(bucket, filename, user_profile, headers: dict): +def try_download_from_bucket(bucket, filename, user_profile, headers: dict, api_request_uuid): timer = Timer() timer.mark() user_id = None @@ -668,7 +677,16 @@ def try_download_from_bucket(bucket, filename, user_profile, headers: dict): redirheaders.update(headers) # Generate URL - presigned_url = get_presigned_url(creds, bucket, filename, bucket_region, expires_in, user_id) + presigned_url = get_presigned_url( + creds, + bucket, + filename, + bucket_region, + expires_in, + user_id, + "GET", + api_request_uuid + ) s3_host = urlparse(presigned_url).netloc log.debug("Presigned URL host was %s", s3_host) @@ -1126,7 +1144,9 @@ def dynamic_url(): log.debug("timing for dynamic_url()") timer.log_all(log) - return try_download_from_bucket(entry.bucket, entry.object_key, user_profile, custom_headers) + api_request_uuid = get_api_request_uuid(app.current_request.query_params) + + return try_download_from_bucket(entry.bucket, entry.object_key, user_profile, custom_headers, api_request_uuid) @app.route("/s3credentials", methods=["GET"])