From 6703a918b202c26304b13b1ef209d6b9aad92204 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 11 Dec 2023 10:48:29 -0800 Subject: [PATCH] Add initial multi-client support to Boto Manager --- fence/__init__.py | 3 ++- fence/blueprints/data/indexd.py | 1 + fence/resources/aws/boto_manager.py | 27 +++++++++++++++++++++++---- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/fence/__init__.py b/fence/__init__.py index 2d44e33f2..1fcc4b24e 100755 --- a/fence/__init__.py +++ b/fence/__init__.py @@ -389,7 +389,8 @@ def app_config( def _setup_data_endpoint_and_boto(app): if "AWS_CREDENTIALS" in config and len(config["AWS_CREDENTIALS"]) > 0: value = list(config["AWS_CREDENTIALS"].values())[0] - app.boto = BotoManager(value, logger=logger) + buckets = config.get("S3_BUCKETS", {}) + app.boto = BotoManager(value, buckets, logger=logger) app.register_blueprint(fence.blueprints.data.blueprint, url_prefix="/data") diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index ca88d8c64..96cb4d21c 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1150,6 +1150,7 @@ def complete_multipart_upload(self, uploadId, parts, expires_in): def delete(self, bucket, file_id): try: + print(f"DEBUG boto: {flask.current_app.boto}") return flask.current_app.boto.delete_data_file(bucket, file_id) except Exception as e: logger.error(e) diff --git a/fence/resources/aws/boto_manager.py b/fence/resources/aws/boto_manager.py index f9ac792f2..ebac01071 100644 --- a/fence/resources/aws/boto_manager.py +++ b/fence/resources/aws/boto_manager.py @@ -17,13 +17,31 @@ class BotoManager(object): 900 # minimum time for aws assume role is 900 seconds as per boto docs ) - def __init__(self, config, logger): + def __init__(self, config, buckets, logger): self.sts_client = client("sts", **config) - self.s3_client = client("s3", endpoint_url='TODO', **config) + self.s3_client = client("s3", **config) + self.s3_clients = self.create_s3_clients(config, buckets) self.logger = logger self.ec2 = None self.iam = None + def create_s3_clients(self, config, buckets): + s3_clients = { + 'default': client('s3', **config) + } + for bucket in buckets: + print(f"DEBUG bucket: {bucket}") + if buckets[bucket]['endpoint_url'] is not None: + print(f"DEBUG endpoint_url: {endpoint_url}") + endpoint_url = buckets[bucket]['endpoint_url'] + s3_clients[bucket] = client('s3', **config, endpoint_url=endpoint_url) + return s3_clients + + def get_s3_client(self, bucket): + if self.s3_clients.get(bucket) is None: + return self.s3_clients['default'] + return self.s3_clients[bucket] + def delete_data_file(self, bucket, prefix): """ We use buckets with versioning disabled. @@ -33,7 +51,8 @@ def delete_data_file(self, bucket, prefix): https://docs.aws.amazon.com/AmazonS3/latest/dev/DeletingObjectsfromVersioningSuspendedBuckets.html """ try: - s3_objects = self.s3_client.list_objects_v2( + s3_client = self.get_s3_client(bucket) + s3_objects = s3_client.list_objects_v2( Bucket=bucket, Prefix=prefix, Delimiter="/" ) @@ -52,7 +71,7 @@ def delete_data_file(self, bucket, prefix): self.logger.error("multiple files found with prefix {}".format(prefix)) return ("Multiple files found matching this prefix. Backing off.", 400) key = s3_objects["Contents"][0]["Key"] - self.s3_client.delete_object(Bucket=bucket, Key=key) + s3_client.delete_object(Bucket=bucket, Key=key) self.logger.info( "deleted file for prefix {} in bucket {}".format(prefix, bucket) )