log unknown S3 key for debugging

eu-cdse/openeo-cdse-infra#55
Open-EO · Feb 13, 2024 · 1c13f11 · 1c13f11
1 parent cb41af0
commit 1c13f11
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 3 deletions.
diff --git a/openeo_driver/views.py b/openeo_driver/views.py
@@ -1214,11 +1214,11 @@ def _download_job_result(
     def _stream_from_s3(s3_url, result, bytes_range: Optional[str]):
         import botocore.exceptions
 
-        bucket, folder = s3_url[5:].split("/", 1)
+        bucket, key = s3_url[5:].split("/", 1)
         s3_instance = _s3_client()
 
         try:
-            s3_file_object = s3_instance.get_object(Bucket=bucket, Key=folder, **dict_no_none(Range=bytes_range))
+            s3_file_object = s3_instance.get_object(Bucket=bucket, Key=key, **dict_no_none(Range=bytes_range))
             body = s3_file_object["Body"]
             resp = flask.Response(
                 response=body.iter_chunks(STREAM_CHUNK_SIZE_DEFAULT),
@@ -1228,14 +1228,17 @@ def _stream_from_s3(s3_url, result, bytes_range: Optional[str]):
             resp.headers['Accept-Ranges'] = 'bytes'
             resp.headers['Content-Length'] = s3_file_object['ContentLength']
             return resp
+        except s3_instance.exceptions.NoSuchKey:
+            _log.exception(f"No such key: s3://{bucket}/{key}")
+            raise
         except botocore.exceptions.ClientError as e:
             if e.response.get("ResponseMetadata").get("HTTPStatusCode") == 416:  # best effort really
                 raise OpenEOApiException(
                     code="RangeNotSatisfiable", status_code=416,
                     message=f"Invalid Range {bytes_range}"
                 )
 
-            raise e
+            raise
 
     @api_endpoint
     @blueprint.route('/jobs/<job_id>/results/items/<user_base64>/<secure_key>/<item_id>', methods=['GET'])

diff --git a/tests/test_views.py b/tests/test_views.py
@@ -2465,6 +2465,30 @@ def test_download_result_with_s3_object_storage_with_expiration_supports_range_r
                                             headers={'Range': "bytes=8-10"})
             out_of_range_get_resp.assert_status_code(416)
 
+    @mock.patch("time.time", mock.MagicMock(return_value=1234))
+    @pytest.mark.parametrize("backend_config_overrides", [{"url_signer": UrlSigner(secret="123&@#", expiration=1000)}])
+    def test_download_result_with_s3_object_storage_with_expiration_NoSuchKey_is_logged(
+            self, api, mock_s3_resource, backend_config_overrides, caplog):
+        job_id = "07024ee9-7847-4b8a-b260-6c879a2b3cdc"
+        s3_bucket_name = "openeo-test-bucket"
+        output_root = f"s3://{s3_bucket_name}/some-data-dir"
+        s3_key = f"some-data-dir/{job_id}/output.tiff"
+
+        s3_bucket = create_s3_bucket(mock_s3_resource, s3_bucket_name)
+
+        jobs = {job_id: {"status": "finished"}}
+        with self._fresh_job_registry(output_root=output_root, jobs=jobs):
+            full_get_resp = api.get('/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA%3D%3D/fd0ca65e29c6d223da05b2e73a875683/output.tiff?expires=2234')
+            # should return a 500 response
+            # should have logged an ERROR
+
+        assert (full_get_resp.assert_status_code(500).json["message"]
+                == "Server error: NoSuchKey('An error occurred (NoSuchKey) when calling the GetObject operation:"
+                   " The specified key does not exist.')")
+
+        assert ("openeo_driver.views", logging.ERROR,
+                f"No such key: s3://{s3_bucket_name}/{s3_key}") in caplog.record_tuples
+
     @mock.patch("time.time", mock.MagicMock(return_value=3456))
     @pytest.mark.parametrize("backend_config_overrides", [{"url_signer": UrlSigner(secret="123&@#", expiration=1000)}])
     def test_download_result_signed_with_expiration_invalid(self, api, tmp_path, flask_app, backend_config_overrides):