From b0ced7df6201276811ae2bb3094df6ae322b629a Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Thu, 1 Feb 2024 10:45:56 +0000 Subject: [PATCH 1/2] Add an endpoint to stream document data --- Pipfile | 2 +- Pipfile.lock | 261 ++++++++++++++---- api/documents/libraries/s3_operations.py | 25 +- ...{test_views.py => test_document_detail.py} | 3 +- api/documents/tests/test_document_stream.py | 94 +++++++ api/documents/urls.py | 1 + api/documents/views.py | 20 +- 7 files changed, 345 insertions(+), 61 deletions(-) rename api/documents/tests/{test_views.py => test_document_detail.py} (96%) create mode 100644 api/documents/tests/test_document_stream.py diff --git a/Pipfile b/Pipfile index 6dc291f1f6..6f90ce8e58 100644 --- a/Pipfile +++ b/Pipfile @@ -20,6 +20,7 @@ ipdb = "*" watchdog = {extras = ["watchmedo"], version = "*"} diff-pdf-visually = "~=1.7.0" pytest-circleci-parallelized = "~=0.1.0" +moto = {extras = ["s3"], version = "*"} [packages] factory-boy = "~=2.12.0" @@ -72,7 +73,6 @@ django-silk = "~=5.0.3" django = "~=4.2.8" django-queryable-properties = "~=1.9.1" - [requires] python_version = "3.8" python_full_version = "3.8.18" diff --git a/Pipfile.lock b/Pipfile.lock index 38f66e45e7..237f40855a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "04ae25b9b6e86cf3b2f10aa738e98c1e31218e1de6de25656d01d8912cb2c917" + "sha256": "75167ed3a0c211bb03d4b612649496882904d5fe8fa30fb6885db3fd49c7a7ad" }, "pipfile-spec": 6, "requires": { @@ -25,14 +25,6 @@ "markers": "python_version >= '3.6'", "version": "==5.2.0" }, - "appnope": { - "hashes": [ - "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24", - "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e" - ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.3" - }, "asgiref": { "hashes": [ "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e", @@ -654,7 +646,7 @@ "hashes": [ "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.18.3" }, "gevent": { @@ -773,7 +765,7 @@ "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da", "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33" ], - "markers": "platform_python_implementation == 'CPython' and python_version < '3.11'", + "markers": "python_version < '3.11' and platform_python_implementation == 'CPython'", "version": "==3.0.3" }, "gunicorn": { @@ -948,11 +940,11 @@ }, "phonenumbers": { "hashes": [ - "sha256:ad7bc7d7fd6599a124423ffb840409630777c72d0ee58ba8070cc8e7efcb4c38", - "sha256:e22f276b0c4a70bd5b3f6d668d19cab2578f660b8df44d6418f81d64320151b9" + "sha256:9d7863dc8a37e8127f3c9dde65be93a5b46649b779184f8b0a85bdd043b0b293", + "sha256:a6c85b53e28410aba2f312255cc8015f384a43e7e241ffb84ca5cde80f094cdf" ], "index": "pypi", - "version": "==8.13.28" + "version": "==8.13.29" }, "pickleshare": { "hashes": [ @@ -1032,7 +1024,6 @@ "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48", "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868" ], - "index": "pypi", "markers": "python_version >= '3.8'", "version": "==10.2.0" }, @@ -1191,15 +1182,15 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, "pytz": { "hashes": [ - "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b", - "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7" + "sha256:31d4583c4ed539cd037956140d695e42c033a19e984bfce9964a3f7d59bc2b40", + "sha256:f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a" ], - "version": "==2023.3.post1" + "version": "==2023.4" }, "redis": { "hashes": [ @@ -1264,7 +1255,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "sqlparse": { @@ -1464,14 +1455,6 @@ } }, "develop": { - "appnope": { - "hashes": [ - "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24", - "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e" - ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.3" - }, "astroid": { "hashes": [ "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", @@ -1525,6 +1508,23 @@ "markers": "python_full_version >= '3.6.2'", "version": "==22.3.0" }, + "boto3": { + "hashes": [ + "sha256:9e7242b9059d937f34264125fecd844cb5e01acce6be093f6c44869fdf7c6e30", + "sha256:fa85b67147c8dc99b6e7c699fc086103f958f9677db934f70659e6e6a72a818c" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==1.26.165" + }, + "botocore": { + "hashes": [ + "sha256:6f35d59e230095aed7cd747604fe248fa384bebb7d09549077892f936a8ca3df", + "sha256:988b948be685006b43c4bbd8f5c0cb93e77c66deb70561994e0c5b31b5a67210" + ], + "markers": "python_version >= '3.7'", + "version": "==1.29.165" + }, "certifi": { "hashes": [ "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", @@ -1533,6 +1533,64 @@ "markers": "python_version >= '3.6'", "version": "==2023.11.17" }, + "cffi": { + "hashes": [ + "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc", + "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a", + "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417", + "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab", + "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520", + "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36", + "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743", + "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8", + "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed", + "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684", + "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56", + "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324", + "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d", + "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235", + "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e", + "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088", + "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000", + "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7", + "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e", + "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673", + "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c", + "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe", + "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2", + "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098", + "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8", + "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a", + "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0", + "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b", + "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896", + "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e", + "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9", + "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2", + "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b", + "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6", + "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404", + "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f", + "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0", + "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4", + "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc", + "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936", + "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba", + "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872", + "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb", + "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614", + "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1", + "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d", + "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969", + "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b", + "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4", + "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627", + "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", + "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" + ], + "markers": "python_version >= '3.8'", + "version": "==1.16.0" + }, "charset-normalizer": { "hashes": [ "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", @@ -1696,6 +1754,36 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==5.5" }, + "cryptography": { + "hashes": [ + "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960", + "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a", + "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc", + "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a", + "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf", + "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1", + "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39", + "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406", + "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a", + "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a", + "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c", + "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be", + "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15", + "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2", + "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d", + "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157", + "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003", + "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248", + "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a", + "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec", + "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309", + "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7", + "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==41.0.7" + }, "decorator": { "hashes": [ "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", @@ -1810,6 +1898,22 @@ "markers": "python_version >= '3.6'", "version": "==0.19.1" }, + "jinja2": { + "hashes": [ + "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa", + "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90" + ], + "markers": "python_version >= '3.7'", + "version": "==3.1.3" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, "lazy-object-proxy": { "hashes": [ "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", @@ -1839,11 +1943,11 @@ }, "mako": { "hashes": [ - "sha256:463f03e04559689adaee25e0967778d6ad41285ed607dc1e7df0dd4e4df81f9e", - "sha256:baee30b9c61718e093130298e678abed0dbfa1b411fcc4c1ab4df87cd631a0f2" + "sha256:2a0c8ad7f6274271b3bb7467dd37cf9cc6dab4bc19cb69a4ef10669402de698e", + "sha256:32a99d70754dfce237019d17ffe4a282d2d3351b9c476e90d8a60e63f133b80c" ], "markers": "python_version >= '3.8'", - "version": "==1.3.1" + "version": "==1.3.2" }, "markupsafe": { "hashes": [ @@ -1926,6 +2030,17 @@ ], "version": "==0.6.1" }, + "moto": { + "extras": [ + "s3" + ], + "hashes": [ + "sha256:1d01de681da1453335ec09ba43db521e577cbd58d25ddfb61e5965534b8be539", + "sha256:4a94a147ee70e85e0842da8d1093728c66085165775d1d302f0f77538bf92b95" + ], + "markers": "python_version >= '3.8'", + "version": "==5.0.0" + }, "mypy-extensions": { "hashes": [ "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", @@ -1954,17 +2069,17 @@ }, "parse": { "hashes": [ - "sha256:5e171b001452fa9f004c5a58a93525175468daf69b493e9fa915347ed7ff6968", - "sha256:bd28bae37714b45d5894d77160a16e2be36b64a3b618c81168b3684676aa498b" + "sha256:09002ca350ad42e76629995f71f7b518670bcf93548bdde3684fd55d2be51975", + "sha256:76ddd5214255ae711db4c512be636151fbabaa948c6f30115aecc440422ca82c" ], - "version": "==1.20.0" + "version": "==1.20.1" }, "parse-type": { "hashes": [ "sha256:06d39a8b70fde873eb2a131141a0e79bb34a432941fb3d66fad247abafc9766c", "sha256:79b1f2497060d0928bc46016793f1fca1057c4aacdf15ef876aa48d75a73a355" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.6.2" }, "parso": { @@ -2015,19 +2130,19 @@ }, "platformdirs": { "hashes": [ - "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380", - "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420" + "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068", + "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768" ], "markers": "python_version >= '3.8'", - "version": "==4.1.0" + "version": "==4.2.0" }, "pluggy": { "hashes": [ - "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12", - "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7" + "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981", + "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be" ], "markers": "python_version >= '3.8'", - "version": "==1.3.0" + "version": "==1.4.0" }, "prompt-toolkit": { "hashes": [ @@ -2051,6 +2166,13 @@ ], "version": "==0.7.0" }, + "py-partiql-parser": { + "hashes": [ + "sha256:427a662e87d51a0a50150fc8b75c9ebb4a52d49129684856c40c88b8c8e027e4", + "sha256:dc454c27526adf62deca5177ea997bf41fac4fd109c5d4c8d81f984de738ba8f" + ], + "version": "==0.5.0" + }, "pycodestyle": { "hashes": [ "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f", @@ -2059,6 +2181,13 @@ "markers": "python_version >= '3.8'", "version": "==2.11.1" }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, "pydocstyle": { "hashes": [ "sha256:118762d452a49d6b05e194ef344a55822987a462831ade91ec5c06fd2169d019", @@ -2128,12 +2257,12 @@ }, "pytest": { "hashes": [ - "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280", - "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8" + "sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c", + "sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6" ], "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==7.4.4" + "markers": "python_version >= '3.8'", + "version": "==8.0.0" }, "pytest-bdd": { "hashes": [ @@ -2163,19 +2292,19 @@ }, "pytest-django": { "hashes": [ - "sha256:4e1c79d5261ade2dd58d91208017cd8f62cb4710b56e012ecd361d15d5d662a2", - "sha256:92d6fd46b1d79b54fb6b060bbb39428073396cec717d5f2e122a990d4b6aa5e8" + "sha256:5d054fe011c56f3b10f978f41a8efb2e5adfc7e680ef36fb571ada1f24779d90", + "sha256:ca1ddd1e0e4c227cf9e3e40a6afc6d106b3e70868fd2ac5798a22501271cd0c7" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.7.0" + "version": "==4.8.0" }, "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, "pyyaml": { @@ -2257,6 +2386,22 @@ ], "version": "==0.7" }, + "responses": { + "hashes": [ + "sha256:a2b43f4c08bfb9c9bd242568328c65a34b318741d3fab884ac843c5ceeb543f9", + "sha256:b127c6ca3f8df0eb9cc82fd93109a3007a86acb24871834c47b77765152ecf8c" + ], + "markers": "python_version >= '3.8'", + "version": "==0.24.1" + }, + "s3transfer": { + "hashes": [ + "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084", + "sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.2" + }, "setoptconf": { "hashes": [ "sha256:1fa613dc4a6fbfbaab9a52319d1e369d030e8ed80455b151574ccf3390ec86c6", @@ -2278,7 +2423,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "smmap": { @@ -2379,11 +2524,27 @@ ], "version": "==0.2.13" }, + "werkzeug": { + "hashes": [ + "sha256:507e811ecea72b18a404947aded4b3390e1db8f826b494d76550ef45bb3b1dcc", + "sha256:90a285dc0e42ad56b34e696398b8122ee4c681833fb35b8334a095d82c56da10" + ], + "markers": "python_version >= '3.8'", + "version": "==3.0.1" + }, "wrapt": { "hashes": [ "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" ], "version": "==1.11.2" + }, + "xmltodict": { + "hashes": [ + "sha256:50d8c638ed7ecb88d90561beedbf720c9b4e851a9fa6c47ebd64e99d166d8a21", + "sha256:8bbcb45cc982f48b2ca8fe7e7827c5d792f217ecf1792626f808bf41c3b86051" + ], + "index": "pypi", + "version": "==0.12.0" } } } diff --git a/api/documents/libraries/s3_operations.py b/api/documents/libraries/s3_operations.py index a99a20a1b7..178a810717 100644 --- a/api/documents/libraries/s3_operations.py +++ b/api/documents/libraries/s3_operations.py @@ -17,13 +17,24 @@ AWS_STORAGE_BUCKET_NAME, ) -_client = boto3.client( - "s3", - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - region_name=AWS_REGION, - config=Config(connect_timeout=S3_CONNECT_TIMEOUT, read_timeout=S3_REQUEST_TIMEOUT), -) + +_client = None + + +def init_s3_client(): + # We want to instantiate this once, ideally, but there may be cases where we + # want to explicitly re-instiate the client e.g. in tests. + global _client + _client = boto3.client( + "s3", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + region_name=AWS_REGION, + config=Config(connect_timeout=S3_CONNECT_TIMEOUT, read_timeout=S3_REQUEST_TIMEOUT), + ) + + +init_s3_client() def get_object(document_id, s3_key): diff --git a/api/documents/tests/test_views.py b/api/documents/tests/test_document_detail.py similarity index 96% rename from api/documents/tests/test_views.py rename to api/documents/tests/test_document_detail.py index edeb1fc5d3..81990c7ec6 100644 --- a/api/documents/tests/test_views.py +++ b/api/documents/tests/test_document_detail.py @@ -1,10 +1,9 @@ from django.urls import reverse -from api.documents import permissions from test_helpers.clients import DataTestClient -class CertificateDownload(DataTestClient): +class DocumentDetail(DataTestClient): def test_document_detail_as_caseworker(self): # given there is a case document case = self.create_standard_application_case(self.organisation) diff --git a/api/documents/tests/test_document_stream.py b/api/documents/tests/test_document_stream.py new file mode 100644 index 0000000000..33fd1bec21 --- /dev/null +++ b/api/documents/tests/test_document_stream.py @@ -0,0 +1,94 @@ +import boto3 + +from moto import mock_aws + +from django.http import StreamingHttpResponse +from django.urls import reverse + +from test_helpers.clients import DataTestClient + +from api.conf.settings import ( + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + AWS_REGION, + AWS_STORAGE_BUCKET_NAME, +) +from api.documents.libraries.s3_operations import init_s3_client + + +@mock_aws +class DocumentStream(DataTestClient): + def setUp(self): + super().setUp() + init_s3_client() + s3 = boto3.client( + "s3", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + region_name=AWS_REGION, + ) + s3.create_bucket( + Bucket=AWS_STORAGE_BUCKET_NAME, + CreateBucketConfiguration={ + "LocationConstraint": AWS_REGION, + }, + ) + s3.put_object( + Bucket=AWS_STORAGE_BUCKET_NAME, + Key="thisisakey", + Body=b"test", + ) + + def test_document_stream_as_caseworker(self): + # given there is a case document + case = self.create_standard_application_case(self.organisation) + document = self.create_case_document(case=case, user=self.gov_user, name="jimmy") + + # when a caseworker tries to access it + url = reverse("documents:document_stream", kwargs={"pk": document.pk}) + response = self.client.get(url, **self.gov_headers) + + # then they can + self.assertEqual(response.status_code, 200) + self.assertIsInstance(response, StreamingHttpResponse) + self.assertEqual(b"".join(response.streaming_content), b"test") + + def test_document_stream_as_exporter(self): + # given there is a case document that is visible to the exporter + case = self.create_standard_application_case(self.organisation) + document = self.create_case_document(case=case, user=self.gov_user, name="jimmy") + + # when the exporter tries to access it + url = reverse("documents:document_stream", kwargs={"pk": document.pk}) + response = self.client.get(url, **self.exporter_headers) + + # then they can + self.assertEqual(response.status_code, 200) + self.assertIsInstance(response, StreamingHttpResponse) + self.assertEqual(b"".join(response.streaming_content), b"test") + + def test_document_stream_as_exporter_on_invisible_document(self): + # givem there is a document that's invisible to the exporter + case = self.create_standard_application_case(self.organisation) + document = self.create_case_document(case=case, user=self.gov_user, name="jimmy", visible_to_exporter=False) + + # when the exporter tries to access it + url = reverse("documents:document_stream", kwargs={"pk": document.pk}) + response = self.client.get(url, **self.exporter_headers) + + # then they cannot + self.assertEqual(response.status_code, 403) + + def test_document_stream_as_illegal_exporter(self): + # given there is a case document in organisation a + other_organisation, _ = self.create_organisation_with_exporter_user() + case = self.create_standard_application_case(other_organisation) + document = self.create_case_document(case=case, user=self.gov_user, name="jimmy", visible_to_exporter=False) + + url = reverse("documents:document_stream", kwargs={"pk": document.pk}) + + # when user from organisation b tries to access it + response = self.client.get(url, **self.exporter_headers) + + # then they are not able to + self.assertEqual(response.status_code, 403) diff --git a/api/documents/urls.py b/api/documents/urls.py index 19430254bf..023beee070 100644 --- a/api/documents/urls.py +++ b/api/documents/urls.py @@ -7,4 +7,5 @@ urlpatterns = [ path("/", views.DocumentDetail.as_view(), name="document"), path("certificate/", views.DownloadSigningCertificate.as_view(), name="certificate"), + path("stream//", views.DocumentStream.as_view(), name="document_stream"), ] diff --git a/api/documents/views.py b/api/documents/views.py index 2714bdd107..8c622e9d00 100644 --- a/api/documents/views.py +++ b/api/documents/views.py @@ -1,12 +1,16 @@ from rest_framework.views import APIView from rest_framework.generics import RetrieveAPIView -from django.http import JsonResponse, HttpResponse +from django.http import ( + HttpResponse, + JsonResponse, +) from django.shortcuts import Http404 from api.cases.generated_documents.signing import get_certificate_data from api.core.authentication import SharedAuthentication from api.core.exceptions import NotFoundError +from api.documents.libraries.s3_operations import document_download_stream from api.documents.models import Document from api.documents.serializers import DocumentViewSerializer from api.documents import permissions @@ -41,3 +45,17 @@ def get(self, request): response = HttpResponse(content=certificate, content_type="application/x-x509-ca-cert") response["Content-Disposition"] = f'attachment; filename="LITECertificate.crt"' return response + + +class DocumentStream(RetrieveAPIView): + """ + Get streamed content of a document. + """ + + authentication_classes = (SharedAuthentication,) + queryset = Document.objects.all() + permission_classes = (permissions.IsCaseworkerOrInDocumentOrganisation,) + + def retrieve(self, request, *args, **kwargs): + document = self.get_object() + return document_download_stream(document) From c1a36aedd3c365abb175597d5522671992deabb7 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Mon, 5 Feb 2024 14:59:57 +0000 Subject: [PATCH 2/2] Only serve safe files from document streaming endpoint --- api/documents/tests/test_document_stream.py | 24 +++++++++++++++++++++ api/documents/views.py | 2 +- test_helpers/clients.py | 4 ++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/api/documents/tests/test_document_stream.py b/api/documents/tests/test_document_stream.py index 33fd1bec21..e613770c85 100644 --- a/api/documents/tests/test_document_stream.py +++ b/api/documents/tests/test_document_stream.py @@ -92,3 +92,27 @@ def test_document_stream_as_illegal_exporter(self): # then they are not able to self.assertEqual(response.status_code, 403) + + def test_document_stream_unsafe_file_as_caseworker(self): + # given there is a case document + case = self.create_standard_application_case(self.organisation) + document = self.create_case_document(case=case, user=self.gov_user, name="jimmy", safe=False) + + # when a caseworker tries to access it + url = reverse("documents:document_stream", kwargs={"pk": document.pk}) + response = self.client.get(url, **self.gov_headers) + + # then they can + self.assertEqual(response.status_code, 404) + + def test_document_stream_unsafe_file_as_exporter(self): + # given there is a case document that is visible to the exporter + case = self.create_standard_application_case(self.organisation) + document = self.create_case_document(case=case, user=self.gov_user, name="jimmy", safe=False) + + # when the exporter tries to access it + url = reverse("documents:document_stream", kwargs={"pk": document.pk}) + response = self.client.get(url, **self.exporter_headers) + + # then they can + self.assertEqual(response.status_code, 404) diff --git a/api/documents/views.py b/api/documents/views.py index 8c622e9d00..c8820a77a7 100644 --- a/api/documents/views.py +++ b/api/documents/views.py @@ -53,7 +53,7 @@ class DocumentStream(RetrieveAPIView): """ authentication_classes = (SharedAuthentication,) - queryset = Document.objects.all() + queryset = Document.objects.filter(safe=True) permission_classes = (permissions.IsCaseworkerOrInDocumentOrganisation,) def retrieve(self, request, *args, **kwargs): diff --git a/test_helpers/clients.py b/test_helpers/clients.py index e2236aaa64..c684f2d6ea 100644 --- a/test_helpers/clients.py +++ b/test_helpers/clients.py @@ -348,7 +348,7 @@ def submit_application(application: BaseApplication, user: ExporterUser = None): return application @staticmethod - def create_case_document(case: Case, user: GovUser, name: str, visible_to_exporter=True): + def create_case_document(case: Case, user: GovUser, name: str, visible_to_exporter=True, safe=True): case_doc = CaseDocument( case=case, description="This is a document", @@ -357,7 +357,7 @@ def create_case_document(case: Case, user: GovUser, name: str, visible_to_export s3_key="thisisakey", size=123456, virus_scanned_at=None, - safe=None, + safe=safe, visible_to_exporter=visible_to_exporter, ) case_doc.save()