diff --git a/.secrets.baseline b/.secrets.baseline index 9aa531c19..82a2c6b2a 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -268,14 +268,14 @@ "filename": "tests/conftest.py", "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", "is_verified": false, - "line_number": 1569 + "line_number": 1570 }, { "type": "Base64 High Entropy String", "filename": "tests/conftest.py", "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108", "is_verified": false, - "line_number": 1593 + "line_number": 1594 } ], "tests/credentials/google/test_credentials.py": [ @@ -422,5 +422,5 @@ } ] }, - "generated_at": "2024-07-25T17:19:58Z" + "generated_at": "2024-08-22T19:43:39Z" } diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index fe1383321..81abfee98 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1,6 +1,8 @@ import re import time import json +import boto3 +from botocore.client import Config from urllib.parse import urlparse, ParseResult, urlunparse from datetime import datetime, timedelta @@ -8,9 +10,9 @@ from cached_property import cached_property import gen3cirrus from gen3cirrus import GoogleCloudManager +from gen3cirrus import AwsService from cdislogging import get_logger from cdispyutils.config import get_value -from cdispyutils.hmac4 import generate_aws_presigned_url import flask from flask import current_app import requests @@ -396,7 +398,7 @@ def make_signed_url(self, file_name, protocol=None, expires_in=None, bucket=None @staticmethod def init_multipart_upload(key, expires_in=None, bucket=None): """ - Initilize multipart upload given key + Initialize multipart upload given key Args: key(str): object key @@ -441,7 +443,7 @@ def generate_aws_presigned_url_for_part( Args: key(str): object key of `guid/filename` - uploadID(str): uploadId of the current upload. + uploadId(str): uploadId of the current upload. partNumber(int): the part number Returns: @@ -1061,6 +1063,8 @@ def get_signed_url( bucket_name = self.bucket_name() bucket = s3_buckets.get(bucket_name) + object_id = self.parsed_url.path.strip("/") + if bucket and bucket.get("endpoint_url"): http_url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( self.parsed_url.netloc, self.parsed_url.path.strip("/") @@ -1092,18 +1096,38 @@ def get_signed_url( region = flask.current_app.boto.get_bucket_region( self.parsed_url.netloc, credential ) + s3client = boto3.client( + "s3", + aws_access_key_id=credential["aws_access_key_id"], + aws_secret_access_key=credential["aws_secret_access_key"], + aws_session_token=credential.get("aws_session_token", None), + region_name=region, + config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), + ) + cirrus_aws = AwsService(s3client) auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) - url = generate_aws_presigned_url( - http_url, - ACTION_DICT["s3"][action], - credential, - "s3", - region, - expires_in, - auth_info, - ) + action = ACTION_DICT["s3"][action] + + # get presigned url for upload + if action == "PUT": + url = cirrus_aws.upload_presigned_url( + bucket_name, object_id, expires_in, None + ) + # get presigned url for download + else: + if bucket.get("requester_pays") is True: + # need to add extra parameter to signing url for header + # https://github.com/boto/boto3/issues/3685 + auth_info["x-amz-request-payer"] = "requester" + url = cirrus_aws.requester_pays_download_presigned_url( + bucket_name, object_id, expires_in, auth_info + ) + else: + url = cirrus_aws.download_presigned_url( + bucket_name, object_id, expires_in, auth_info + ) return url @@ -1115,7 +1139,7 @@ def init_multipart_upload(self, expires_in): expires(int): expiration time Returns: - UploadId(str) + uploadId(str) """ aws_creds = get_value( config, "AWS_CREDENTIALS", InternalError("credentials not configured") @@ -1133,18 +1157,19 @@ def generate_presigned_url_for_part_upload(self, uploadId, partNumber, expires_i Generate presigned url for uploading object part given uploadId and part number Args: - uploadId(str): uploadID of the multipart upload + uploadId(str): uploadId of the multipart upload partNumber(int): part number expires(int): expiration time Returns: presigned_url(str) """ + bucket_name = self.bucket_name() aws_creds = get_value( config, "AWS_CREDENTIALS", InternalError("credentials not configured") ) credential = S3IndexedFileLocation.get_credential_to_access_bucket( - self.bucket_name(), aws_creds, expires_in + bucket_name, aws_creds, expires_in ) region = self.get_bucket_region() @@ -1154,7 +1179,7 @@ def generate_presigned_url_for_part_upload(self, uploadId, partNumber, expires_i ) return multipart_upload.generate_presigned_url_for_uploading_part( - self.parsed_url.netloc, + bucket_name, self.parsed_url.path.strip("/"), credential, uploadId, diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index 7352f66f1..fd271317b 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -1,10 +1,11 @@ import boto3 +from botocore.client import Config from botocore.exceptions import ClientError from retry.api import retry_call -from cdispyutils.hmac4 import generate_aws_presigned_url from cdispyutils.config import get_value from cdislogging import get_logger +from gen3cirrus import AwsService from fence.config import config from fence.errors import InternalError @@ -58,7 +59,7 @@ def initialize_multipart_upload(bucket_name, key, credentials): key, error ) ) - raise InternalError("Can not initilize multipart upload for {}".format(key)) + raise InternalError("Can not initialize multipart upload for {}".format(key)) return multipart_upload.get("UploadId") @@ -140,28 +141,21 @@ def generate_presigned_url_for_uploading_part( Returns: presigned_url(str) """ - s3_buckets = get_value( - config, "S3_BUCKETS", InternalError("S3_BUCKETS not configured") - ) - bucket = s3_buckets.get(bucket_name) - - s3_buckets = get_value( - config, "S3_BUCKETS", InternalError("S3_BUCKETS not configured") - ) - bucket = s3_buckets.get(bucket_name) - - if bucket.get("endpoint_url"): - url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( - bucket_name, key.strip("/") + try: + s3client = boto3.client( + "s3", + aws_access_key_id=credentials["aws_access_key_id"], + aws_secret_access_key=credentials["aws_secret_access_key"], + aws_session_token=credentials.get("aws_session_token", None), + region_name=region, + config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), ) - else: - url = "https://{}.s3.amazonaws.com/{}".format(bucket_name, key) - additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId} + cirrus_aws = AwsService(s3client) - try: - presigned_url = generate_aws_presigned_url( - url, "PUT", credentials, "s3", region, expires, additional_signed_qs + presigned_url = cirrus_aws.multipart_upload_presigned_url( + bucket_name, key, expires, uploadId, partNumber ) + return presigned_url except Exception as e: raise InternalError( diff --git a/fence/config-default.yaml b/fence/config-default.yaml index 5e43e21dc..a570989c0 100755 --- a/fence/config-default.yaml +++ b/fence/config-default.yaml @@ -676,6 +676,10 @@ S3_BUCKETS: {} # cred: 'CRED1' # region: 'us-east-1' # role-arn: 'arn:aws:iam::role1' +# bucket5: +# cred: 'CRED3' +# region: 'us-east-1' +# requester_pays: true # to indicate this is a requester pay enabled S3 bucket GS_BUCKETS: {} # NOTE: Remove the {} and supply buckets if needed. Example in comments below # bucket1: diff --git a/poetry.lock b/poetry.lock index 29e85be0a..a91239b85 100644 --- a/poetry.lock +++ b/poetry.lock @@ -250,17 +250,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.158" +version = "1.35.6" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.158-py3-none-any.whl", hash = "sha256:c29e9b7e1034e8734ccaffb9f2b3f3df2268022fd8a93d836604019f8759ce27"}, - {file = "boto3-1.34.158.tar.gz", hash = "sha256:5b7b2ce0ec1e498933f600d29f3e1c641f8c44dd7e468c26795359d23d81fa39"}, + {file = "boto3-1.35.6-py3-none-any.whl", hash = "sha256:c35c560ef0cb0f133b6104bc374d60eeb7cb69c1d5d7907e4305a285d162bef0"}, + {file = "boto3-1.35.6.tar.gz", hash = "sha256:b41deed9ca7e0a619510a22e256e3e38b5f532624b4aff8964a1e870877b37bc"}, ] [package.dependencies] -botocore = ">=1.34.158,<1.35.0" +botocore = ">=1.35.6,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -269,13 +269,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.158" +version = "1.35.6" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.158-py3-none-any.whl", hash = "sha256:0e6fceba1e39bfa8feeba70ba3ac2af958b3387df4bd3b5f2db3f64c1754c756"}, - {file = "botocore-1.34.158.tar.gz", hash = "sha256:5934082e25ad726673afbf466092fb1223dafa250e6e756c819430ba6b1b3da5"}, + {file = "botocore-1.35.6-py3-none-any.whl", hash = "sha256:8378c6cfef2dee15eb7b3ebbb55ba9c1de959f231292039b81eb35b72c50ad59"}, + {file = "botocore-1.35.6.tar.gz", hash = "sha256:93ef31b80b05758db4dd67e010348a05b9ff43f82839629b7ac334f2a454996e"}, ] [package.dependencies] @@ -313,13 +313,13 @@ files = [ [[package]] name = "cachetools" -version = "5.4.0" +version = "5.5.0" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.4.0-py3-none-any.whl", hash = "sha256:3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474"}, - {file = "cachetools-5.4.0.tar.gz", hash = "sha256:b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827"}, + {file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"}, + {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, ] [[package]] @@ -961,17 +961,18 @@ six = ">=1.16.0,<2.0.0" [[package]] name = "gen3cirrus" -version = "3.0.1" +version = "3.1.0" description = "" optional = false -python-versions = ">=3.9,<4.0" +python-versions = "<4.0,>=3.9" files = [ - {file = "gen3cirrus-3.0.1-py3-none-any.whl", hash = "sha256:74628faca3b1cbe65c78e08eb567e1ac0cb8ae52e1bfc603f904af0277e3cb52"}, - {file = "gen3cirrus-3.0.1.tar.gz", hash = "sha256:0ae0ddc0ee7df870603457fe186245f3c8124d989254276e5011a23e1139a6c8"}, + {file = "gen3cirrus-3.1.0-py3-none-any.whl", hash = "sha256:42c89d1579d7d89c87c5c355815197e1dbf8045a2030310a6b2f6ca089a74fde"}, + {file = "gen3cirrus-3.1.0.tar.gz", hash = "sha256:81e5a0a4b5dc2d820ad3351bd5326151424806bc8f64e022100320f01027a310"}, ] [package.dependencies] backoff = "*" +boto3 = "*" cdislogging = "*" google-api-python-client = "*" google-auth = "*" @@ -1038,13 +1039,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.140.0" +version = "2.142.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "google_api_python_client-2.140.0-py2.py3-none-any.whl", hash = "sha256:aeb4bb99e9fdd241473da5ff35464a0658fea0db76fe89c0f8c77ecfc3813404"}, - {file = "google_api_python_client-2.140.0.tar.gz", hash = "sha256:0bb973adccbe66a3d0a70abe4e49b3f2f004d849416bfec38d22b75649d389d8"}, + {file = "google_api_python_client-2.142.0-py2.py3-none-any.whl", hash = "sha256:266799082bb8301f423ec204dffbffb470b502abbf29efd1f83e644d36eb5a8f"}, + {file = "google_api_python_client-2.142.0.tar.gz", hash = "sha256:a1101ac9e24356557ca22f07ff48b7f61fa5d4b4e7feeef3bda16e5dcb86350e"}, ] [package.dependencies] @@ -1056,13 +1057,13 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.33.0" +version = "2.34.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google_auth-2.33.0-py2.py3-none-any.whl", hash = "sha256:8eff47d0d4a34ab6265c50a106a3362de6a9975bb08998700e389f857e4d39df"}, - {file = "google_auth-2.33.0.tar.gz", hash = "sha256:d6a52342160d7290e334b4d47ba390767e4438ad0d45b7630774533e82655b95"}, + {file = "google_auth-2.34.0-py2.py3-none-any.whl", hash = "sha256:72fd4733b80b6d777dcde515628a9eb4a577339437012874ea286bca7261ee65"}, + {file = "google_auth-2.34.0.tar.gz", hash = "sha256:8eb87396435c19b20d32abd2f984e31c191a15284af72eb922f10e5bde9c04cc"}, ] [package.dependencies] @@ -1072,7 +1073,7 @@ rsa = ">=3.1.4,<5" [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] -enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +enterprise-cert = ["cryptography", "pyopenssl"] pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] requests = ["requests (>=2.20.0,<3.0.0.dev0)"] @@ -1233,13 +1234,13 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.63.2" +version = "1.64.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.63.2.tar.gz", hash = "sha256:27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87"}, - {file = "googleapis_common_protos-1.63.2-py2.py3-none-any.whl", hash = "sha256:27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945"}, + {file = "googleapis_common_protos-1.64.0-py2.py3-none-any.whl", hash = "sha256:d1bfc569f70ed2e96ccf06ead265c2cf42b5abfc817cda392e3835f3b67b5c59"}, + {file = "googleapis_common_protos-1.64.0.tar.gz", hash = "sha256:7d77ca6b7c0c38eb6b1bab3b4c9973acf57ce4f2a6d3a4136acba10bcbfb3025"}, ] [package.dependencies] @@ -1367,13 +1368,13 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 [[package]] name = "httpx" -version = "0.27.0" +version = "0.27.2" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpx-0.27.0-py3-none-any.whl", hash = "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5"}, - {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"}, + {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, + {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, ] [package.dependencies] @@ -1388,27 +1389,28 @@ brotli = ["brotli", "brotlicffi"] cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "idna" -version = "3.7" +version = "3.8" description = "Internationalized Domain Names in Applications (IDNA)" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, + {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, ] [[package]] name = "importlib-metadata" -version = "8.2.0" +version = "8.4.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-8.2.0-py3-none-any.whl", hash = "sha256:11901fa0c2f97919b288679932bb64febaeacf289d18ac84dd68cb2e74213369"}, - {file = "importlib_metadata-8.2.0.tar.gz", hash = "sha256:72e8d4399996132204f9a16dcc751af254a48f8d1b20b9ff0f98d4a8f901e73d"}, + {file = "importlib_metadata-8.4.0-py3-none-any.whl", hash = "sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1"}, + {file = "importlib_metadata-8.4.0.tar.gz", hash = "sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5"}, ] [package.dependencies] @@ -1419,6 +1421,17 @@ doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linke perf = ["ipython"] test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "isodate" version = "0.6.1" @@ -1519,13 +1532,13 @@ testing = ["pytest"] [[package]] name = "markdown" -version = "3.6" +version = "3.7" description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.8" files = [ - {file = "Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f"}, - {file = "Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"}, + {file = "Markdown-3.7-py3-none-any.whl", hash = "sha256:7eb6df5690b81a1d7942992c97fad2938e956e79df20cbc6186e9c3a77b1c803"}, + {file = "markdown-3.7.tar.gz", hash = "sha256:2ae2471477cfd02dbbf038d5d9bc226d40def84b4fe2986e49b59b6b472bbed2"}, ] [package.dependencies] @@ -1623,17 +1636,6 @@ six = ">=1.9" docs = ["Pygments (<2)", "jinja2 (<2.7)", "sphinx", "sphinx (<1.3)"] test = ["unittest2 (>=1.1.0)"] -[[package]] -name = "more-itertools" -version = "10.4.0" -description = "More routines for operating on iterables, beyond itertools" -optional = false -python-versions = ">=3.8" -files = [ - {file = "more-itertools-10.4.0.tar.gz", hash = "sha256:fe0e63c4ab068eac62410ab05cccca2dc71ec44ba8ef29916a0090df061cf923"}, - {file = "more_itertools-10.4.0-py3-none-any.whl", hash = "sha256:0f7d9f83a0a8dcfa8a2694a770590d98a67ea943e3d9f5298309a484758c4e27"}, -] - [[package]] name = "moto" version = "1.3.7" @@ -1699,13 +1701,13 @@ files = [ [[package]] name = "paramiko" -version = "3.4.0" +version = "3.4.1" description = "SSH2 protocol library" optional = false python-versions = ">=3.6" files = [ - {file = "paramiko-3.4.0-py3-none-any.whl", hash = "sha256:43f0b51115a896f9c00f59618023484cb3a14b98bbceab43394a39c6739b7ee7"}, - {file = "paramiko-3.4.0.tar.gz", hash = "sha256:aac08f26a31dc4dffd92821527d1682d99d52f9ef6851968114a8728f3c274d3"}, + {file = "paramiko-3.4.1-py3-none-any.whl", hash = "sha256:8e49fd2f82f84acf7ffd57c64311aa2b30e575370dc23bdb375b10262f7eac32"}, + {file = "paramiko-3.4.1.tar.gz", hash = "sha256:8b15302870af7f6652f2e038975c1d2973f06046cb5d7d65355668b3ecbece0c"}, ] [package.dependencies] @@ -1720,28 +1722,29 @@ invoke = ["invoke (>=2.0)"] [[package]] name = "pbr" -version = "6.0.0" +version = "6.1.0" description = "Python Build Reasonableness" optional = false python-versions = ">=2.6" files = [ - {file = "pbr-6.0.0-py2.py3-none-any.whl", hash = "sha256:4a7317d5e3b17a3dccb6a8cfe67dab65b20551404c52c8ed41279fa4f0cb4cda"}, - {file = "pbr-6.0.0.tar.gz", hash = "sha256:d1377122a5a00e2f940ee482999518efe16d745d423a670c27773dfbc3c9a7d9"}, + {file = "pbr-6.1.0-py2.py3-none-any.whl", hash = "sha256:a776ae228892d8013649c0aeccbb3d5f99ee15e005a4cbb7e61d55a067b28a2a"}, + {file = "pbr-6.1.0.tar.gz", hash = "sha256:788183e382e3d1d7707db08978239965e8b9e4e5ed42669bf4758186734d5f24"}, ] [[package]] name = "pluggy" -version = "0.13.1" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.8" files = [ - {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, - {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [package.extras] dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "prometheus-client" @@ -1969,13 +1972,13 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] name = "pyparsing" -version = "3.1.2" +version = "3.1.4" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.6.8" files = [ - {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"}, - {file = "pyparsing-3.1.2.tar.gz", hash = "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad"}, + {file = "pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c"}, + {file = "pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"}, ] [package.extras] @@ -1983,27 +1986,26 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "5.4.3" +version = "6.2.5" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, - {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"}, + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" +attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} -more-itertools = ">=4.0.0" +iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" -py = ">=1.5.0" -wcwidth = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" [package.extras] -checkqa-mypy = ["mypy (==v0.761)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -2440,26 +2442,15 @@ files = [ cdislogging = "*" sqlalchemy = ">=1.3.3" -[[package]] -name = "wcwidth" -version = "0.2.13" -description = "Measures the displayed width of unicode strings in a terminal" -optional = false -python-versions = "*" -files = [ - {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, - {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, -] - [[package]] name = "werkzeug" -version = "3.0.3" +version = "3.0.4" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.8" files = [ - {file = "werkzeug-3.0.3-py3-none-any.whl", hash = "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"}, - {file = "werkzeug-3.0.3.tar.gz", hash = "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18"}, + {file = "werkzeug-3.0.4-py3-none-any.whl", hash = "sha256:02c9eb92b7d6c06f31a782811505d2157837cea66aaede3e217c7c27c039476c"}, + {file = "werkzeug-3.0.4.tar.gz", hash = "sha256:34f2371506b250df4d4f84bfe7b0921e4762525762bbd936614909fe25cd7306"}, ] [package.dependencies] @@ -2577,20 +2568,24 @@ files = [ [[package]] name = "zipp" -version = "3.19.2" +version = "3.20.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, - {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, + {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, + {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, ] [package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0.0" -content-hash = "d003418dcc0d68257a215186d21776941f8739bd3b4f898762a93e7895d7c89e" +content-hash = "8c9ff3ba536d69f449fb04dab58808da7996c3449472c751f2b6da24394d7305" diff --git a/pyproject.toml b/pyproject.toml index 2a8b74a29..9086fa0ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fence" -version = "10.2.0" +version = "10.3.0" description = "Gen3 AuthN/AuthZ OIDC Service" authors = ["CTDS UChicago "] license = "Apache-2.0" @@ -13,7 +13,7 @@ include = [ [tool.poetry.dependencies] python = ">=3.9,<4.0.0" alembic = "^1.7.7" -authlib = "*" #let authutils decide which version to use +authlib = "<1.3.2" # let authutils decide which version to use, but 1.3.2 will cause some unit tests to fail authutils = "^6.2.3" bcrypt = "^3.1.4" boto3 = "*" @@ -28,7 +28,7 @@ flask-cors = ">=3.0.3" flask-restful = ">=0.3.8" email_validator = "^1.1.1" gen3authz = "^1.5.1" -gen3cirrus = ">=3.0.1" +gen3cirrus = "^3.1.0" gen3config = ">=1.1.0" gen3users = "^1.0.2" idna = "^3.7" @@ -39,7 +39,7 @@ markupsafe = "^2.0.1" paramiko = ">=2.6.0" prometheus-client = "<1" -psycopg2 = "^2.8.3" +psycopg2 = "<3" PyJWT = "^2.4.0" python_dateutil = "^2.6.1" python-jose = "^2.0.2" @@ -64,7 +64,7 @@ codacy-coverage = "^1.3.11" coveralls = "^2.1.1" mock = "^2.0.0" moto = "^1.1.24" -pytest = "^5.2.0" +pytest = "^6.2.5" pytest-cov = "^2.5.1" pytest-flask = ">=1.3.0" diff --git a/tests/conftest.py b/tests/conftest.py index 273f4a496..9baba01a1 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,6 +34,7 @@ import requests from sqlalchemy.ext.compiler import compiles + # Set FENCE_CONFIG_PATH *before* loading the configuration CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) os.environ["FENCE_CONFIG_PATH"] = os.path.join(CURRENT_DIR, "test-fence-config.yaml") @@ -1596,6 +1597,36 @@ def google_signed_url(): return manager +@pytest.fixture(scope="function") +def aws_signed_url(): + """ + Mock signed urls coming from AWS using a side effect function + """ + + def presigned_url_side_effect(*args, **kwargs): + additional_qs = "" + if args[3] and isinstance(args[3], dict): + for k in args[3]: + additional_qs += f"{k}={args[3][k]}&" + return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}&{additional_qs}" + + manager = MagicMock(side_effect=presigned_url_side_effect) + + down = patch( + "fence.blueprints.data.indexd.gen3cirrus.aws.services.AwsService.download_presigned_url", + manager, + ).start() + up = patch( + "fence.blueprints.data.indexd.gen3cirrus.aws.services.AwsService.upload_presigned_url", + manager, + ).start() + + yield manager + + down.stop() + up.stop() + + @pytest.fixture(scope="function") def encoded_creds_jwt( kid, rsa_private_key, user_client, oauth_client, google_proxy_group diff --git a/tests/data/test_blank_index.py b/tests/data/test_blank_index.py index 7586d85be..f5f6bb400 100755 --- a/tests/data/test_blank_index.py +++ b/tests/data/test_blank_index.py @@ -41,7 +41,14 @@ def text(self): return self.data -def test_blank_index_upload(app, client, auth_client, encoded_creds_jwt, user_client): +def test_blank_index_upload( + app, + client, + auth_client, + encoded_creds_jwt, + user_client, + aws_signed_url, +): """ test BlankIndex upload POST /data/upload @@ -149,6 +156,7 @@ def test_blank_index_upload_bucket( user_client, bucket, expected_status_code, + aws_signed_url, ): """ Same test as above, except request a specific bucket to upload the file to diff --git a/tests/data/test_data.py b/tests/data/test_data.py index 568fb97bd..f0a3a9852 100755 --- a/tests/data/test_data.py +++ b/tests/data/test_data.py @@ -72,6 +72,7 @@ def test_indexd_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -123,6 +124,7 @@ def test_indexd_upload_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -163,6 +165,7 @@ def test_indexd_upload_file_key_error( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test upload with a missing configuration key should fail @@ -211,6 +214,7 @@ def test_indexd_upload_file_filename( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, guid, file_name, ): @@ -255,6 +259,7 @@ def test_indexd_upload_file_filename_key_error( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1?file_name=`` with an example file name @@ -310,6 +315,7 @@ def test_indexd_upload_file_bucket( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, bucket, expected_status_code, ): @@ -353,6 +359,7 @@ def test_indexd_upload_file_doesnt_exist( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1`` when 1 doesn't exist. @@ -391,6 +398,7 @@ def test_indexd_download_file_no_protocol( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -443,6 +451,7 @@ def test_indexd_unauthorized_download_file( indexd_client, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -472,6 +481,7 @@ def test_unauthorized_indexd_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -536,6 +546,7 @@ def test_unauthorized_indexd_upload_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1``. @@ -600,6 +611,7 @@ def test_unavailable_indexd_upload_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1``. @@ -660,6 +672,7 @@ def test_public_object_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -686,6 +699,7 @@ def test_public_object_download_file_no_force_sign( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1?no_force_sign=True``. @@ -721,6 +735,7 @@ def test_public_bucket_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1`` with public bucket @@ -755,6 +770,7 @@ def test_public_bucket_download_file_no_force_sign( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1`` with public bucket with no_force_sign request @@ -777,6 +793,7 @@ def test_public_bucket_unsupported_protocol_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1`` with public bucket @@ -1271,6 +1288,7 @@ def test_assume_role_cache( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1`` with authorized user (user is the uploader). @@ -1380,6 +1398,7 @@ def test_indexd_download_with_uploader_unauthorized( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1`` with unauthorized user (user is not the uploader). @@ -1603,7 +1622,12 @@ def json(self): def test_blank_index_upload_unauthorized( - app, client, auth_client, encoded_creds_jwt, user_client + app, + client, + auth_client, + encoded_creds_jwt, + user_client, + aws_signed_url, ): class MockResponse(object): def __init__(self, data, status_code=200): @@ -1650,6 +1674,7 @@ def test_abac( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): mock_arborist_requests({"arborist/auth/request": {"POST": ({"auth": True}, 200)}}) indexd_client = indexd_client_with_arborist("test_abac")