From 7c0db1003d53e576b4e63e4cf62937066dfdc8d4 Mon Sep 17 00:00:00 2001 From: jreadey Date: Fri, 26 Apr 2024 09:10:10 -0700 Subject: [PATCH] support s3fs for anonymous creds --- examples/read_example.py | 2 +- h5pyd/_apps/hsload.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/examples/read_example.py b/examples/read_example.py index 4decd1c..3f32f53 100755 --- a/examples/read_example.py +++ b/examples/read_example.py @@ -36,7 +36,7 @@ def load_file(): print("running command:", " ".join(run_cmd)) result = subprocess.run(run_cmd) if result.returncode != 0: - print(f"unable able to hsload {s3_uri}, error: {run_cmd.returncode}") + print(f"unable able to hsload {s3_uri}, error: {result.returncode}") sys.exit(1) print("hsload complete") # now we should be able to open the domain diff --git a/h5pyd/_apps/hsload.py b/h5pyd/_apps/hsload.py index 8fb72ee..f56ea78 100755 --- a/h5pyd/_apps/hsload.py +++ b/h5pyd/_apps/hsload.py @@ -192,14 +192,24 @@ def main(): sys.exit(1) if not s3: + kwargs = {"use_ssl": False} key = os.environ.get("AWS_ACCESS_KEY_ID") secret = os.environ.get("AWS_SECRET_ACCESS_KEY") - aws_s3_gateway = os.environ.get("AWS_GATEWAY") + + if not key or not secret: + kwargs["anon"] = True + else: + kwargs["key"] = key + kwargs["secret"] = secret + client_kwargs = {} + aws_s3_gateway = os.environ.get("AWS_GATEWAY") if aws_s3_gateway: client_kwargs["endpoint_url"] = aws_s3_gateway + if client_kwargs: + kwargs["client_kwargs"] = client_kwargs - s3 = s3fs.S3FileSystem(use_ssl=False, key=key, secret=secret, client_kwargs=client_kwargs) + s3 = s3fs.S3FileSystem(**kwargs) try: fin = h5py.File(s3.open(src_file, "rb"), moe="r") except IOError as ioe: