From 74d38bf4265a2efc00b829c05de2fcddfde7f973 Mon Sep 17 00:00:00 2001 From: jreadey Date: Mon, 14 Aug 2023 18:47:15 +0200 Subject: [PATCH] use tcp by default in hsds app --- hsds/app.py | 91 +++++++++++++++++++++++++------------------ hsds/config.py | 14 ++++++- hsds/domain_sn.py | 5 --- hsds/hsds_app.py | 2 + hsds/util/s3Client.py | 1 + tests/integ/config.py | 2 - 6 files changed, 69 insertions(+), 46 deletions(-) diff --git a/hsds/app.py b/hsds/app.py index a88b3a2d..4394f567 100644 --- a/hsds/app.py +++ b/hsds/app.py @@ -15,24 +15,33 @@ import sys import logging import time -import uuid from .hsds_app import HsdsApp +from . import config _HELP_USAGE = "Starts hsds a REST-based service for HDF5 data." _HELP_EPILOG = """Examples: +- with a POSIX-based storage using a directory: ./hsdata for storage: + + hsds --root_dir ~/hsdata + +- with POSIX-based storage and config settings and password file: + + hsds --root_dir ~/hsdata --password-file ./admin/config/passwd.txt \ + --config_dir ./admin/config + - with minio data storage: hsds --s3-gateway http://localhost:6007 --access-key-id demo:demo --secret-access-key DEMO_PASS --password-file ./admin/config/passwd.txt - --bucket-name hsds.test -- with a POSIX-based storage for 'hsds.test' sub-folder in the './data' - folder: +- with AWS S3 storage and a bucket in the us-west-2 region: + + hsds --s3-gateway http://s3.us-west-2.amazonaws.com --access-key-id ${AWS_ACCESS_KEY_ID} \ + --secret-access-key ${AWS_SECRET_ACCESS_KEY} --password-file ./admin/config/passwd.txt - hsds --bucket-dir ./data/hsds.test """ # maximum number of characters if socket directory is given @@ -139,14 +148,13 @@ def main(): epilog=_HELP_EPILOG, ) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( + parser.add_argument( "--root_dir", type=str, dest="root_dir", help="Directory where to store the object store data", ) - group.add_argument( + parser.add_argument( "--bucket_name", nargs=1, type=str, @@ -197,7 +205,7 @@ def main(): ) parser.add_argument( "--count", - default=1, + default=4, type=int, dest="dn_count", help="Number of dn sub-processes to create.", @@ -251,6 +259,9 @@ def main(): username = args.hs_username elif "HS_USERNAME" in userConfig: username = userConfig["HS_USERNAME"] + elif not args.password_file: + # no password file, add the login name as user + username = os.getlogin() else: username = None @@ -260,7 +271,7 @@ def main(): elif "HS_PASSWORD" in userConfig: password = userConfig["HS_PASSWORD"] else: - password = "1234" + password = os.getlogin() if username: kwargs["username"] = username @@ -271,38 +282,23 @@ def main(): sys.exit(f"password file: {args.password_file} not found") kwargs["password_file"] = args.password_file - if args.host: - # use TCP connect - kwargs["host"] = args.host + # use unix domain socket if a socket dir is set + if args.socket_dir: + socket_dir = os.path.abspath(args.socket_dir) + if not os.path.isdir(socket_dir): + raise FileNotFoundError(f"directory: {socket_dir} not found") + kwargs["socket_dir"] = socket_dir + else: + # USE TCP connect + if args.host: + kwargs["host"] = args.host + else: + kwargs["host"] = "localhost" # sn_port only relevant for TCP connections if args.port: kwargs["sn_port"] = args.port else: kwargs["sn_port"] = 5101 # TBD - use config - else: - # choose a tmp directory for socket if one is not provided - if args.socket_dir: - socket_dir = os.path.abspath(args.socket_dir) - if not os.path.isdir(socket_dir): - raise FileNotFoundError(f"directory: {socket_dir} not found") - else: - if "TMP" in os.environ: - # This should be set at least on Windows - tmp_dir = os.environ["TMP"] - print("set tmp_dir:", tmp_dir) - else: - tmp_dir = "/tmp" - if not os.path.isdir(tmp_dir): - raise FileNotFoundError(f"directory {tmp_dir} not found") - rand_name = uuid.uuid4().hex[:8] - socket_dir = os.path.join(tmp_dir, f"hs{rand_name}") - print("using socket dir:", socket_dir) - if len(socket_dir) > MAX_SOCKET_DIR_PATH_LEN: - raise ValueError( - f"length of socket_dir must be less than: {MAX_SOCKET_DIR_PATH_LEN}" - ) - os.mkdir(socket_dir) - kwargs["socket_dir"] = socket_dir if args.logfile: logfile = os.path.abspath(args.logfile) @@ -329,6 +325,27 @@ def main(): if args.dn_count: kwargs["dn_count"] = args.dn_count + if args.bucket_name: + bucket_name = args.bucket_name + else: + bucket_name = config.get("bucket_name") + if not bucket_name: + sys.exit("bucket_name not set") + if args.root_dir: + root_dir = args.root_dir + else: + root_dir = config.get("root_dir") + if not root_dir: + # check that AWS_S3_GATEWAY or AZURE_CONNECTION_STRING is set + if not config.get("aws_s3_gateway") and not config.get("azure_connection_string"): + sys.exit("root_dir not set (and no S3 or Azure connection info)") + else: + if not os.path.isdir(root_dir): + sys.exit(f"directory: {root_dir} not found") + bucket_path = os.path.join(root_dir, bucket_name) + if not os.path.isdir(bucket_path): + os.mkdir(bucket_path) + app = HsdsApp(**kwargs) app.run() diff --git a/hsds/config.py b/hsds/config.py index d640c276..799f6036 100755 --- a/hsds/config.py +++ b/hsds/config.py @@ -42,13 +42,21 @@ def getCmdLineArg(x): # return value of command-line option # use "--x=val" to set option 'x' to 'val' # use "--x" for boolean flags + option = "--" + x + "=" for i in range(1, len(sys.argv)): arg = sys.argv[i] + if i < len(sys.argv) - 1: + next_arg = sys.argv[i + 1] + else: + next_arg = None if arg == "--" + x: - # boolean flag debug(f"got cmd line flag for {x}") - return True + if next_arg is None or next_arg.startswith("-"): + # treat as a boolean flag + return True + else: + return next_arg elif arg.startswith(option): # found an override nlen = len(option) @@ -69,6 +77,7 @@ def _load_cfg(): config_dir = getCmdLineArg("config_dir") if config_dir: + eprint("got command line arg for config_dir:", config_dir) config_dirs.append(config_dir) if not config_dirs and "CONFIG_DIR" in os.environ: config_dirs.append(os.environ["CONFIG_DIR"]) @@ -77,6 +86,7 @@ def _load_cfg(): debug("set default location for config dirs") config_dirs = ["./", "/config", "/etc/hsds/"] # default locations for config_dir in config_dirs: + eprint("using config_dir:", config_dir) file_name = os.path.join(config_dir, "config.yml") debug("checking config path:", file_name) if os.path.isfile(file_name): diff --git a/hsds/domain_sn.py b/hsds/domain_sn.py index b0bf5323..ee82e28b 100755 --- a/hsds/domain_sn.py +++ b/hsds/domain_sn.py @@ -1123,11 +1123,6 @@ async def PUT_Domain(request): else: is_toplevel = False - if is_toplevel and not is_folder: - msg = "Only folder domains can be created at the top-level" - log.warn(msg) - raise HTTPBadRequest(reason=msg) - if is_toplevel and not isAdminUser(app, username): msg = "creation of top-level domains is only supported by admin users" log.warn(msg) diff --git a/hsds/hsds_app.py b/hsds/hsds_app.py index 1c5740ce..5924d722 100644 --- a/hsds/hsds_app.py +++ b/hsds/hsds_app.py @@ -274,6 +274,8 @@ def run(self): pargs = [py_exe, cmd_path, "--node_type=sn", "--log_prefix=sn "] if self._username: pargs.append(f"--hs_username={self._username}") + # make this user admin + pargs.append(f"--admin_user={self._username}") if self._password: pargs.append(f"--hs_password={self._password}") if self._password_file: diff --git a/hsds/util/s3Client.py b/hsds/util/s3Client.py index 9b1b44ba..f2ddd7cc 100644 --- a/hsds/util/s3Client.py +++ b/hsds/util/s3Client.py @@ -145,6 +145,7 @@ def _get_client_kwargs(self): kwargs["endpoint_url"] = self._s3_gateway kwargs["use_ssl"] = self._use_ssl kwargs["config"] = self._aio_config + log.debug(f"s3 kwargs: {kwargs}") return kwargs def _renewToken(self): diff --git a/tests/integ/config.py b/tests/integ/config.py index f8ad3194..5b23b9aa 100755 --- a/tests/integ/config.py +++ b/tests/integ/config.py @@ -14,8 +14,6 @@ cfg = { "hsds_endpoint": "http://localhost:5101", # or 'http+unix://%2Ftmp%2Fhs%2Fsn_1.sock' for socket - "head_endpoint": "http://localhost:5100", - "rangeget_endpoint": "http://localhost:6900", "user_name": "test_user1", "user_password": "test", "user2_name": "test_user2",