diff --git a/Pipfile.lock b/Pipfile.lock index 237f40855a..3d7c9dbfdb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -133,11 +133,11 @@ }, "certifi": { "hashes": [ - "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", - "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474" + "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", + "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" ], "markers": "python_version >= '3.6'", - "version": "==2023.11.17" + "version": "==2024.2.2" }, "cffi": { "hashes": [ @@ -1024,6 +1024,7 @@ "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48", "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868" ], + "index": "pypi", "markers": "python_version >= '3.8'", "version": "==10.2.0" }, @@ -1187,10 +1188,10 @@ }, "pytz": { "hashes": [ - "sha256:31d4583c4ed539cd037956140d695e42c033a19e984bfce9964a3f7d59bc2b40", - "sha256:f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a" + "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812", + "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319" ], - "version": "==2023.4" + "version": "==2024.1" }, "redis": { "hashes": [ @@ -1527,11 +1528,11 @@ }, "certifi": { "hashes": [ - "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", - "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474" + "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", + "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" ], "markers": "python_version >= '3.6'", - "version": "==2023.11.17" + "version": "==2024.2.2" }, "cffi": { "hashes": [ @@ -1951,69 +1952,69 @@ }, "markupsafe": { "hashes": [ - "sha256:0042d6a9880b38e1dd9ff83146cc3c9c18a059b9360ceae207805567aacccc69", - "sha256:0c26f67b3fe27302d3a412b85ef696792c4a2386293c53ba683a89562f9399b0", - "sha256:0fbad3d346df8f9d72622ac71b69565e621ada2ce6572f37c2eae8dacd60385d", - "sha256:15866d7f2dc60cfdde12ebb4e75e41be862348b4728300c36cdf405e258415ec", - "sha256:1c98c33ffe20e9a489145d97070a435ea0679fddaabcafe19982fe9c971987d5", - "sha256:21e7af8091007bf4bebf4521184f4880a6acab8df0df52ef9e513d8e5db23411", - "sha256:23984d1bdae01bee794267424af55eef4dfc038dc5d1272860669b2aa025c9e3", - "sha256:31f57d64c336b8ccb1966d156932f3daa4fee74176b0fdc48ef580be774aae74", - "sha256:3583a3a3ab7958e354dc1d25be74aee6228938312ee875a22330c4dc2e41beb0", - "sha256:36d7626a8cca4d34216875aee5a1d3d654bb3dac201c1c003d182283e3205949", - "sha256:396549cea79e8ca4ba65525470d534e8a41070e6b3500ce2414921099cb73e8d", - "sha256:3a66c36a3864df95e4f62f9167c734b3b1192cb0851b43d7cc08040c074c6279", - "sha256:3aae9af4cac263007fd6309c64c6ab4506dd2b79382d9d19a1994f9240b8db4f", - "sha256:3ab3a886a237f6e9c9f4f7d272067e712cdb4efa774bef494dccad08f39d8ae6", - "sha256:47bb5f0142b8b64ed1399b6b60f700a580335c8e1c57f2f15587bd072012decc", - "sha256:49a3b78a5af63ec10d8604180380c13dcd870aba7928c1fe04e881d5c792dc4e", - "sha256:4df98d4a9cd6a88d6a585852f56f2155c9cdb6aec78361a19f938810aa020954", - "sha256:5045e892cfdaecc5b4c01822f353cf2c8feb88a6ec1c0adef2a2e705eef0f656", - "sha256:5244324676254697fe5c181fc762284e2c5fceeb1c4e3e7f6aca2b6f107e60dc", - "sha256:54635102ba3cf5da26eb6f96c4b8c53af8a9c0d97b64bdcb592596a6255d8518", - "sha256:54a7e1380dfece8847c71bf7e33da5d084e9b889c75eca19100ef98027bd9f56", - "sha256:55d03fea4c4e9fd0ad75dc2e7e2b6757b80c152c032ea1d1de487461d8140efc", - "sha256:698e84142f3f884114ea8cf83e7a67ca8f4ace8454e78fe960646c6c91c63bfa", - "sha256:6aa5e2e7fc9bc042ae82d8b79d795b9a62bd8f15ba1e7594e3db243f158b5565", - "sha256:7653fa39578957bc42e5ebc15cf4361d9e0ee4b702d7d5ec96cdac860953c5b4", - "sha256:765f036a3d00395a326df2835d8f86b637dbaf9832f90f5d196c3b8a7a5080cb", - "sha256:78bc995e004681246e85e28e068111a4c3f35f34e6c62da1471e844ee1446250", - "sha256:7a07f40ef8f0fbc5ef1000d0c78771f4d5ca03b4953fc162749772916b298fc4", - "sha256:8b570a1537367b52396e53325769608f2a687ec9a4363647af1cded8928af959", - "sha256:987d13fe1d23e12a66ca2073b8d2e2a75cec2ecb8eab43ff5624ba0ad42764bc", - "sha256:9896fca4a8eb246defc8b2a7ac77ef7553b638e04fbf170bff78a40fa8a91474", - "sha256:9e9e3c4020aa2dc62d5dd6743a69e399ce3de58320522948af6140ac959ab863", - "sha256:a0b838c37ba596fcbfca71651a104a611543077156cb0a26fe0c475e1f152ee8", - "sha256:a4d176cfdfde84f732c4a53109b293d05883e952bbba68b857ae446fa3119b4f", - "sha256:a76055d5cb1c23485d7ddae533229039b850db711c554a12ea64a0fd8a0129e2", - "sha256:a76cd37d229fc385738bd1ce4cba2a121cf26b53864c1772694ad0ad348e509e", - "sha256:a7cc49ef48a3c7a0005a949f3c04f8baa5409d3f663a1b36f0eba9bfe2a0396e", - "sha256:abf5ebbec056817057bfafc0445916bb688a255a5146f900445d081db08cbabb", - "sha256:b0fe73bac2fed83839dbdbe6da84ae2a31c11cfc1c777a40dbd8ac8a6ed1560f", - "sha256:b6f14a9cd50c3cb100eb94b3273131c80d102e19bb20253ac7bd7336118a673a", - "sha256:b83041cda633871572f0d3c41dddd5582ad7d22f65a72eacd8d3d6d00291df26", - "sha256:b835aba863195269ea358cecc21b400276747cc977492319fd7682b8cd2c253d", - "sha256:bf1196dcc239e608605b716e7b166eb5faf4bc192f8a44b81e85251e62584bd2", - "sha256:c669391319973e49a7c6230c218a1e3044710bc1ce4c8e6eb71f7e6d43a2c131", - "sha256:c7556bafeaa0a50e2fe7dc86e0382dea349ebcad8f010d5a7dc6ba568eaaa789", - "sha256:c8f253a84dbd2c63c19590fa86a032ef3d8cc18923b8049d91bcdeeb2581fbf6", - "sha256:d18b66fe626ac412d96c2ab536306c736c66cf2a31c243a45025156cc190dc8a", - "sha256:d5291d98cd3ad9a562883468c690a2a238c4a6388ab3bd155b0c75dd55ece858", - "sha256:d5c31fe855c77cad679b302aabc42d724ed87c043b1432d457f4976add1c2c3e", - "sha256:d6e427c7378c7f1b2bef6a344c925b8b63623d3321c09a237b7cc0e77dd98ceb", - "sha256:dac1ebf6983148b45b5fa48593950f90ed6d1d26300604f321c74a9ca1609f8e", - "sha256:de8153a7aae3835484ac168a9a9bdaa0c5eee4e0bc595503c95d53b942879c84", - "sha256:e1a0d1924a5013d4f294087e00024ad25668234569289650929ab871231668e7", - "sha256:e7902211afd0af05fbadcc9a312e4cf10f27b779cf1323e78d52377ae4b72bea", - "sha256:e888ff76ceb39601c59e219f281466c6d7e66bd375b4ec1ce83bcdc68306796b", - "sha256:f06e5a9e99b7df44640767842f414ed5d7bedaaa78cd817ce04bbd6fd86e2dd6", - "sha256:f6be2d708a9d0e9b0054856f07ac7070fbe1754be40ca8525d5adccdbda8f475", - "sha256:f9917691f410a2e0897d1ef99619fd3f7dd503647c8ff2475bf90c3cf222ad74", - "sha256:fc1a75aa8f11b87910ffd98de62b29d6520b6d6e8a3de69a70ca34dea85d2a8a", - "sha256:fe8512ed897d5daf089e5bd010c3dc03bb1bdae00b35588c49b98268d4a01e00" + "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", + "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", + "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", + "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", + "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", + "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", + "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", + "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", + "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", + "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", + "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", + "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", + "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", + "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", + "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", + "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", + "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", + "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", + "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", + "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", + "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", + "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", + "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", + "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", + "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", + "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", + "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", + "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", + "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", + "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", + "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", + "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", + "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", + "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", + "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", + "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", + "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", + "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", + "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", + "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", + "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", + "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", + "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", + "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", + "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", + "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", + "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", + "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", + "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", + "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", + "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", + "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", + "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", + "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", + "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", + "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", + "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", + "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", + "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", + "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" ], "markers": "python_version >= '3.7'", - "version": "==2.1.4" + "version": "==2.1.5" }, "matplotlib-inline": { "hashes": [ @@ -2035,11 +2036,11 @@ "s3" ], "hashes": [ - "sha256:1d01de681da1453335ec09ba43db521e577cbd58d25ddfb61e5965534b8be539", - "sha256:4a94a147ee70e85e0842da8d1093728c66085165775d1d302f0f77538bf92b95" + "sha256:62b9798aef9028432194cebb7a671f4064257bb3be662d9c1b83b94411b694bb", + "sha256:94e3b07a403cc8078ffee94bf404ef677112d036a57ddb5e0f19c5fcf48987f5" ], "markers": "python_version >= '3.8'", - "version": "==5.0.0" + "version": "==5.0.1" }, "mypy-extensions": { "hashes": [ @@ -2168,10 +2169,10 @@ }, "py-partiql-parser": { "hashes": [ - "sha256:427a662e87d51a0a50150fc8b75c9ebb4a52d49129684856c40c88b8c8e027e4", - "sha256:dc454c27526adf62deca5177ea997bf41fac4fd109c5d4c8d81f984de738ba8f" + "sha256:53053e70987dea2983e1990ad85f87a7d8cec13dd4a4b065a740bcfd661f5a6b", + "sha256:aeac8f46529d8651bbae88a1a6c14dc3aa38ebc4bc6bd1eb975044c0564246c6" ], - "version": "==0.5.0" + "version": "==0.5.1" }, "pycodestyle": { "hashes": [ diff --git a/api/cases/tests/test_case_documents.py b/api/cases/tests/test_case_documents.py index a65aa0e85a..af56b478cd 100644 --- a/api/cases/tests/test_case_documents.py +++ b/api/cases/tests/test_case_documents.py @@ -1,13 +1,17 @@ import uuid -from unittest import mock -from django.http import StreamingHttpResponse +from moto import mock_aws + +from django.conf import settings +from django.http import FileResponse from django.urls import reverse from rest_framework import status from lite_content.lite_api.strings import Documents from test_helpers.clients import DataTestClient +from api.documents.libraries.s3_operations import init_s3_client + class CaseDocumentsTests(DataTestClient): def setUp(self): @@ -27,6 +31,7 @@ def test_can_view_all_documents_on_a_case(self): self.assertEqual(len(response_data["documents"]), 2) +@mock_aws class CaseDocumentDownloadTests(DataTestClient): def setUp(self): super().setUp() @@ -35,16 +40,26 @@ def setUp(self): self.file = self.create_case_document(self.case, self.gov_user, "Test") self.path = "cases:document_download" - @mock.patch("api.documents.libraries.s3_operations.get_object") - def test_download_case_document_success(self, get_object_function): - get_object_function.return_value = None + s3 = init_s3_client() + s3.create_bucket( + Bucket=settings.AWS_STORAGE_BUCKET_NAME, + CreateBucketConfiguration={ + "LocationConstraint": settings.AWS_REGION, + }, + ) + s3.put_object( + Bucket=settings.AWS_STORAGE_BUCKET_NAME, + Key=self.file.s3_key, + Body=b"test", + ) + + def test_download_case_document_success(self): url = reverse(self.path, kwargs={"case_pk": self.case.id, "document_pk": self.file.id}) response = self.client.get(url, **self.exporter_headers) - get_object_function.assert_called_once() self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertTrue(isinstance(response, StreamingHttpResponse)) + self.assertTrue(isinstance(response, FileResponse)) self.assertEqual(response.headers["content-disposition"], 'attachment; filename="Test"') def test_download_case_document_invalid_organisation_failure(self): diff --git a/api/conf/settings.py b/api/conf/settings.py index 5529c91a84..71812f98fa 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -238,11 +238,13 @@ raise Exception("S3 Bucket not bound to environment") aws_credentials = VCAP_SERVICES["aws-s3-bucket"][0]["credentials"] + AWS_ENDPOINT_URL = None AWS_ACCESS_KEY_ID = aws_credentials["aws_access_key_id"] AWS_SECRET_ACCESS_KEY = aws_credentials["aws_secret_access_key"] AWS_REGION = aws_credentials["aws_region"] AWS_STORAGE_BUCKET_NAME = aws_credentials["bucket_name"] else: + AWS_ENDPOINT_URL = env("AWS_ENDPOINT_URL", default=None) AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID") AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY") AWS_REGION = env("AWS_REGION") diff --git a/api/conf/settings_test.py b/api/conf/settings_test.py index 925218c3cf..e28fcb845a 100644 --- a/api/conf/settings_test.py +++ b/api/conf/settings_test.py @@ -6,3 +6,5 @@ LOGGING = {"version": 1, "disable_existing_loggers": True} SUPPRESS_TEST_OUTPUT = True + +AWS_ENDPOINT_URL = None diff --git a/api/documents/libraries/s3_operations.py b/api/documents/libraries/s3_operations.py index 178a810717..0942d45151 100644 --- a/api/documents/libraries/s3_operations.py +++ b/api/documents/libraries/s3_operations.py @@ -5,17 +5,9 @@ import boto3 from botocore.config import Config from botocore.exceptions import BotoCoreError, ReadTimeoutError -from django.http import StreamingHttpResponse -from api.conf.settings import ( - STREAMING_CHUNK_SIZE, - S3_CONNECT_TIMEOUT, - S3_REQUEST_TIMEOUT, - AWS_ACCESS_KEY_ID, - AWS_SECRET_ACCESS_KEY, - AWS_REGION, - AWS_STORAGE_BUCKET_NAME, -) +from django.conf import settings +from django.http import FileResponse _client = None @@ -25,13 +17,18 @@ def init_s3_client(): # We want to instantiate this once, ideally, but there may be cases where we # want to explicitly re-instiate the client e.g. in tests. global _client + additional_s3_params = {} + if settings.AWS_ENDPOINT_URL: + additional_s3_params["endpoint_url"] = settings.AWS_ENDPOINT_URL _client = boto3.client( "s3", - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - region_name=AWS_REGION, - config=Config(connect_timeout=S3_CONNECT_TIMEOUT, read_timeout=S3_REQUEST_TIMEOUT), + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + region_name=settings.AWS_REGION, + config=Config(connect_timeout=settings.S3_CONNECT_TIMEOUT, read_timeout=settings.S3_REQUEST_TIMEOUT), + **additional_s3_params, ) + return _client init_s3_client() @@ -41,7 +38,7 @@ def get_object(document_id, s3_key): logging.info(f"Retrieving file '{s3_key}' on document '{document_id}'") try: - return _client.get_object(Bucket=AWS_STORAGE_BUCKET_NAME, Key=s3_key) + return _client.get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=s3_key) except ReadTimeoutError: logging.warning(f"Timeout exceeded when retrieving file '{s3_key}' on document '{document_id}'") except BotoCoreError as exc: @@ -55,14 +52,14 @@ def generate_s3_key(document_name, file_extension): def upload_bytes_file(raw_file, s3_key): - _client.put_object(Bucket=AWS_STORAGE_BUCKET_NAME, Key=s3_key, Body=raw_file) + _client.put_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=s3_key, Body=raw_file) def delete_file(document_id, s3_key): logging.info(f"Deleting file '{s3_key}' on document '{document_id}'") try: - _client.delete_object(Bucket=AWS_STORAGE_BUCKET_NAME, Key=s3_key) + _client.delete_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=s3_key) except ReadTimeoutError: logging.warning(f"Timeout exceeded when retrieving file '{s3_key}' on document '{document_id}'") except BotoCoreError as exc: @@ -71,16 +68,15 @@ def delete_file(document_id, s3_key): ) -def _stream_file(result): - for chunk in iter(lambda: result["Body"].read(STREAMING_CHUNK_SIZE), b""): - yield chunk - - def document_download_stream(document): s3_response = get_object(document.id, document.s3_key) content_type = mimetypes.MimeTypes().guess_type(document.name)[0] - response = StreamingHttpResponse(streaming_content=_stream_file(s3_response), content_type=content_type) - response["Content-Disposition"] = f'attachment; filename="{document.name}"' + response = FileResponse( + s3_response["Body"], + as_attachment=True, + filename=document.name, + ) + response["Content-Type"] = content_type return response diff --git a/api/documents/libraries/tests/__init__.py b/api/documents/libraries/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/documents/libraries/tests/test_s3_operations.py b/api/documents/libraries/tests/test_s3_operations.py new file mode 100644 index 0000000000..53d58e6fb3 --- /dev/null +++ b/api/documents/libraries/tests/test_s3_operations.py @@ -0,0 +1,161 @@ +from contextlib import contextmanager +from unittest.mock import Mock, patch + +from moto import mock_aws + +from django.conf import settings +from django.http import FileResponse +from django.test import override_settings, SimpleTestCase + +from ..s3_operations import ( + delete_file, + document_download_stream, + init_s3_client, + get_object, + upload_bytes_file, +) + + +@patch("api.documents.libraries.s3_operations.boto3") +@patch("api.documents.libraries.s3_operations.Config") +@override_settings( + AWS_ENDPOINT_URL="AWS_ENDPOINT_URL", + AWS_ACCESS_KEY_ID="AWS_ACCESS_KEY_ID", + AWS_SECRET_ACCESS_KEY="AWS_SECRET_ACCESS_KEY", + AWS_REGION="AWS_REGION", + S3_CONNECT_TIMEOUT=22, + S3_REQUEST_TIMEOUT=44, +) +class S3OperationsTests(SimpleTestCase): + @override_settings( + AWS_ENDPOINT_URL=None, + ) + def test_get_client_without_aws_endpoint_url(self, mock_Config, mock_boto3): + mock_client = Mock() + mock_boto3.client.return_value = mock_client + + returned_client = init_s3_client() + self.assertEqual(returned_client, mock_client) + + mock_Config.assert_called_with( + connect_timeout=22, + read_timeout=44, + ) + config = mock_Config( + connection_timeout=22, + read_timeout=44, + ) + mock_boto3.client.assert_called_with( + "s3", + aws_access_key_id="AWS_ACCESS_KEY_ID", + aws_secret_access_key="AWS_SECRET_ACCESS_KEY", + region_name="AWS_REGION", + config=config, + ) + + def test_get_client_with_aws_endpoint_url(self, mock_Config, mock_boto3): + mock_client = Mock() + mock_boto3.client.return_value = mock_client + + returned_client = init_s3_client() + self.assertEqual(returned_client, mock_client) + + mock_Config.assert_called_with( + connect_timeout=22, + read_timeout=44, + ) + config = mock_Config( + connection_timeout=22, + read_timeout=44, + ) + mock_boto3.client.assert_called_with( + "s3", + aws_access_key_id="AWS_ACCESS_KEY_ID", + aws_secret_access_key="AWS_SECRET_ACCESS_KEY", + region_name="AWS_REGION", + config=config, + endpoint_url="AWS_ENDPOINT_URL", + ) + + +@override_settings( + AWS_STORAGE_BUCKET_NAME="test-bucket", +) +class S3OperationsGetObjectTests(SimpleTestCase): + @patch("api.documents.libraries.s3_operations._client") + def test_get_object(self, mock_client): + mock_object = Mock() + mock_client.get_object.return_value = mock_object + + returned_object = get_object("document-id", "s3-key") + + self.assertEqual(returned_object, mock_object) + mock_client.get_object.assert_called_with(Bucket="test-bucket", Key="s3-key") + + +@contextmanager +def _create_bucket(s3): + s3.create_bucket( + Bucket=settings.AWS_STORAGE_BUCKET_NAME, + CreateBucketConfiguration={ + "LocationConstraint": settings.AWS_REGION, + }, + ) + yield + + +@mock_aws +class S3OperationsDeleteFileTests(SimpleTestCase): + def test_delete_file(self): + s3 = init_s3_client() + with _create_bucket(s3): + s3.put_object( + Bucket=settings.AWS_STORAGE_BUCKET_NAME, + Key="s3-key", + Body=b"test", + ) + + delete_file("document-id", "s3-key") + + objs = s3.list_objects(Bucket=settings.AWS_STORAGE_BUCKET_NAME) + keys = [o["Key"] for o in objs.get("Contents", [])] + self.assertNotIn("s3-key", keys) + + +@mock_aws +class S3OperationsUploadBytesFileTests(SimpleTestCase): + def test_upload_bytes_file(self): + s3 = init_s3_client() + with _create_bucket(s3): + upload_bytes_file(b"test", "s3-key") + + obj = s3.get_object( + Bucket=settings.AWS_STORAGE_BUCKET_NAME, + Key="s3-key", + ) + self.assertEqual(obj["Body"].read(), b"test") + + +@mock_aws +class S3OperationsDocumentDownloadStreamTests(SimpleTestCase): + def test_document_download_stream(self): + s3 = init_s3_client() + with _create_bucket(s3): + s3.put_object( + Bucket=settings.AWS_STORAGE_BUCKET_NAME, + Key="s3-key", + Body=b"test", + ) + + mock_document = Mock() + mock_document.id = "document-id" + mock_document.s3_key = "s3-key" + mock_document.name = "test.doc" + + response = document_download_stream(mock_document) + + self.assertIsInstance(response, FileResponse) + self.assertEqual(response.status_code, 200) + self.assertEqual(response["Content-Type"], "application/msword") + self.assertEqual(response["Content-Disposition"], 'attachment; filename="test.doc"') + self.assertEqual(b"".join(response.streaming_content), b"test") diff --git a/api/staticdata/upload_document_for_tests/views.py b/api/staticdata/upload_document_for_tests/views.py index 71bd585c55..bd84fb8749 100644 --- a/api/staticdata/upload_document_for_tests/views.py +++ b/api/staticdata/upload_document_for_tests/views.py @@ -1,12 +1,13 @@ import os -import boto3 +from django.conf import settings from django.http import JsonResponse from rest_framework import status from rest_framework.views import APIView from api.core.authentication import SharedAuthentication -from api.conf.settings import env, AWS_STORAGE_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +from api.conf.settings import env +from api.documents.libraries.s3_operations import init_s3_client class UploadDocumentForTests(APIView): @@ -21,11 +22,7 @@ def get(self, request): status=status.HTTP_405_METHOD_NOT_ALLOWED, ) - s3 = boto3.client( - "s3", - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - ) + s3 = init_s3_client() s3_key = "lite-e2e-test-file.txt" file_to_upload_abs_path = os.path.abspath( @@ -33,7 +30,7 @@ def get(self, request): ) try: - s3.upload_file(file_to_upload_abs_path, AWS_STORAGE_BUCKET_NAME, s3_key) + s3.upload_file(file_to_upload_abs_path, settings.AWS_STORAGE_BUCKET_NAME, s3_key) except Exception as e: # noqa return JsonResponse( data={"errors": "Error uploading file to S3"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/ci.env b/ci.env index ebfe9a9bd6..e2e4f9c6e7 100644 --- a/ci.env +++ b/ci.env @@ -9,6 +9,7 @@ ALLOWED_HOSTS=* INTERNAL_USERS='[{"email": "filippo.raimondi@digital.trade.gov.uk", "role": "Super User"}]' EXPORTER_USERS='[{"email": "filippo.raimondi@digital.trade.gov.uk", "organisation": "Archway Communications", "role": "Super User"}]' # AWS +AWS_ENDPOINT_URL=AWS_ENDPOINT_URL AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY AWS_STORAGE_BUCKET_NAME=AWS_STORAGE_BUCKET_NAME diff --git a/docker-compose.yml b/docker-compose.yml index bebfa19db8..ab4d4dd12e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,6 +98,19 @@ services: networks: - lite + s3: + image: minio/minio + ports: + - 9000:9000 + - 9001:9001 + entrypoint: sh + command: -c 'mkdir -p /buckets/uploads && minio server /buckets --console-address ":9001"' + environment: + - MINIO_ROOT_USER=minio_username + - MINIO_ROOT_PASSWORD=minio_password + networks: + - lite + networks: lite: external: true diff --git a/local.env b/local.env index e62c95c42d..92cdf5f92f 100644 --- a/local.env +++ b/local.env @@ -32,9 +32,10 @@ REDIS_BASE_URL=redis://redis:6379 LITE_API_ENABLE_ES=True # AWS -AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID -AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY -AWS_STORAGE_BUCKET_NAME=AWS_STORAGE_BUCKET_NAME +AWS_ENDPOINT_URL=http://s3:9000 +AWS_ACCESS_KEY_ID=minio_username +AWS_SECRET_ACCESS_KEY=minio_password +AWS_STORAGE_BUCKET_NAME=uploads AWS_REGION=eu-west-2 # AV