use tcp by default in hsds app

hsds/app.py

@@ -15,24 +15,33 @@
 import sys
 import logging
 import time
-import uuid
 
 from .hsds_app import HsdsApp
+from . import config
 
 _HELP_USAGE = "Starts hsds a REST-based service for HDF5 data."
 
 _HELP_EPILOG = """Examples:
 
+- with a POSIX-based storage using a directory: ./hsdata for storage:
+
+  hsds --root_dir ~/hsdata
+
+- with POSIX-based storage and config settings and password file:
+
+  hsds --root_dir ~/hsdata --password-file ./admin/config/passwd.txt \
+  --config_dir ./admin/config
+
 - with minio data storage:
 
   hsds --s3-gateway http://localhost:6007 --access-key-id demo:demo
       --secret-access-key DEMO_PASS --password-file ./admin/config/passwd.txt
-      --bucket-name hsds.test
 
-- with a POSIX-based storage for 'hsds.test' sub-folder in the './data'
-  folder:
+- with AWS S3 storage and a bucket in the us-west-2 region:
+
+  hsds --s3-gateway http://s3.us-west-2.amazonaws.com --access-key-id ${AWS_ACCESS_KEY_ID} \
+      --secret-access-key ${AWS_SECRET_ACCESS_KEY} --password-file ./admin/config/passwd.txt
 
-  hsds --bucket-dir ./data/hsds.test
 """
 
 # maximum number of characters if socket directory is given
@@ -139,14 +148,13 @@ def main():
         epilog=_HELP_EPILOG,
     )
 
-    group = parser.add_mutually_exclusive_group(required=True)
-    group.add_argument(
+    parser.add_argument(
         "--root_dir",
         type=str,
         dest="root_dir",
         help="Directory where to store the object store data",
     )
-    group.add_argument(
+    parser.add_argument(
         "--bucket_name",
         nargs=1,
         type=str,
@@ -197,7 +205,7 @@ def main():
     )
     parser.add_argument(
         "--count",
-        default=1,
+        default=4,
         type=int,
         dest="dn_count",
         help="Number of dn sub-processes to create.",
@@ -251,6 +259,9 @@ def main():
         username = args.hs_username
     elif "HS_USERNAME" in userConfig:
         username = userConfig["HS_USERNAME"]
+    elif not args.password_file:
+        # no password file, add the login name as user
+        username = os.getlogin()
     else:
         username = None
 
@@ -260,7 +271,7 @@ def main():
     elif "HS_PASSWORD" in userConfig:
         password = userConfig["HS_PASSWORD"]
     else:
-        password = "1234"
+        password = os.getlogin()
 
     if username:
         kwargs["username"] = username
@@ -271,38 +282,23 @@ def main():
             sys.exit(f"password file: {args.password_file} not found")
         kwargs["password_file"] = args.password_file
 
-    if args.host:
-        # use TCP connect
-        kwargs["host"] = args.host
+    # use unix domain socket if a socket dir is set
+    if args.socket_dir:
+        socket_dir = os.path.abspath(args.socket_dir)
+        if not os.path.isdir(socket_dir):
+            raise FileNotFoundError(f"directory: {socket_dir} not found")
+        kwargs["socket_dir"] = socket_dir
+    else:
+        # USE TCP connect
+        if args.host:
+            kwargs["host"] = args.host
+        else:
+            kwargs["host"] = "localhost"
         # sn_port only relevant for TCP connections
         if args.port:
             kwargs["sn_port"] = args.port
         else:
             kwargs["sn_port"] = 5101  # TBD - use config
-    else:
-        # choose a tmp directory for socket if one is not provided
-        if args.socket_dir:
-            socket_dir = os.path.abspath(args.socket_dir)
-            if not os.path.isdir(socket_dir):
-                raise FileNotFoundError(f"directory: {socket_dir} not found")
-        else:
-            if "TMP" in os.environ:
-                # This should be set at least on Windows
-                tmp_dir = os.environ["TMP"]
-                print("set tmp_dir:", tmp_dir)
-            else:
-                tmp_dir = "/tmp"
-            if not os.path.isdir(tmp_dir):
-                raise FileNotFoundError(f"directory {tmp_dir} not found")
-            rand_name = uuid.uuid4().hex[:8]
-            socket_dir = os.path.join(tmp_dir, f"hs{rand_name}")
-            print("using socket dir:", socket_dir)
-            if len(socket_dir) > MAX_SOCKET_DIR_PATH_LEN:
-                raise ValueError(
-                    f"length of socket_dir must be less than: {MAX_SOCKET_DIR_PATH_LEN}"
-                )
-            os.mkdir(socket_dir)
-        kwargs["socket_dir"] = socket_dir
 
     if args.logfile:
         logfile = os.path.abspath(args.logfile)
@@ -329,6 +325,27 @@ def main():
     if args.dn_count:
         kwargs["dn_count"] = args.dn_count
 
+    if args.bucket_name:
+        bucket_name = args.bucket_name
+    else:
+        bucket_name = config.get("bucket_name")
+    if not bucket_name:
+        sys.exit("bucket_name not set")
+    if args.root_dir:
+        root_dir = args.root_dir
+    else:
+        root_dir = config.get("root_dir")
+    if not root_dir:
+        # check that AWS_S3_GATEWAY or AZURE_CONNECTION_STRING is set
+        if not config.get("aws_s3_gateway") and not config.get("azure_connection_string"):
+            sys.exit("root_dir not set (and no S3 or Azure connection info)")
+    else:
+        if not os.path.isdir(root_dir):
+            sys.exit(f"directory: {root_dir} not found")
+        bucket_path = os.path.join(root_dir, bucket_name)
+        if not os.path.isdir(bucket_path):
+            os.mkdir(bucket_path)
+
     app = HsdsApp(**kwargs)
     app.run()
 

hsds/config.py

@@ -42,13 +42,21 @@ def getCmdLineArg(x):
     # return value of command-line option
     # use "--x=val" to set option 'x' to 'val'
     # use "--x" for boolean flags
+
     option = "--" + x + "="
     for i in range(1, len(sys.argv)):
         arg = sys.argv[i]
+        if i < len(sys.argv) - 1:
+            next_arg = sys.argv[i + 1]
+        else:
+            next_arg = None
         if arg == "--" + x:
-            # boolean flag
             debug(f"got cmd line flag for {x}")
-            return True
+            if next_arg is None or next_arg.startswith("-"):
+                # treat as a boolean flag
+                return True
+            else:
+                return next_arg
         elif arg.startswith(option):
             # found an override
             nlen = len(option)
@@ -69,6 +77,7 @@ def _load_cfg():
         config_dir = getCmdLineArg("config_dir")
 
     if config_dir:
+        eprint("got command line arg for config_dir:", config_dir)
         config_dirs.append(config_dir)
     if not config_dirs and "CONFIG_DIR" in os.environ:
         config_dirs.append(os.environ["CONFIG_DIR"])
@@ -77,6 +86,7 @@ def _load_cfg():
         debug("set default location for config dirs")
         config_dirs = ["./", "/config", "/etc/hsds/"]  # default locations
     for config_dir in config_dirs:
+        eprint("using config_dir:", config_dir)
         file_name = os.path.join(config_dir, "config.yml")
         debug("checking config path:", file_name)
         if os.path.isfile(file_name):

hsds/domain_sn.py

@@ -1123,11 +1123,6 @@ async def PUT_Domain(request):
     else:
         is_toplevel = False
 
-    if is_toplevel and not is_folder:
-        msg = "Only folder domains can be created at the top-level"
-        log.warn(msg)
-        raise HTTPBadRequest(reason=msg)
-
     if is_toplevel and not isAdminUser(app, username):
         msg = "creation of top-level domains is only supported by admin users"
         log.warn(msg)

hsds/hsds_app.py

@@ -274,6 +274,8 @@ def run(self):
                 pargs = [py_exe, cmd_path, "--node_type=sn", "--log_prefix=sn "]
                 if self._username:
                     pargs.append(f"--hs_username={self._username}")
+                    # make this user admin
+                    pargs.append(f"--admin_user={self._username}")
                 if self._password:
                     pargs.append(f"--hs_password={self._password}")
                 if self._password_file:

hsds/util/s3Client.py

@@ -145,6 +145,7 @@ def _get_client_kwargs(self):
         kwargs["endpoint_url"] = self._s3_gateway
         kwargs["use_ssl"] = self._use_ssl
         kwargs["config"] = self._aio_config
+        log.debug(f"s3 kwargs: {kwargs}")
         return kwargs
 
     def _renewToken(self):

tests/integ/config.py

@@ -14,8 +14,6 @@
 cfg = {
     "hsds_endpoint": "http://localhost:5101",
     # or 'http+unix://%2Ftmp%2Fhs%2Fsn_1.sock' for socket
-    "head_endpoint": "http://localhost:5100",
-    "rangeget_endpoint": "http://localhost:6900",
     "user_name": "test_user1",
     "user_password": "test",
     "user2_name": "test_user2",