diff --git a/Dockerfile b/Dockerfile index 3e881bbf..0de796d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,13 +12,18 @@ RUN mkdir /usr/local/src/hsds/ \ /usr/local/src/hsds/hsds/util/ \ /etc/hsds/ -COPY setup.py /usr/local/src/hsds/ +COPY pyproject.toml /usr/local/src/hsds/ +COPY setup.cfg /user/local/src/hsds/ COPY hsds/*.py /usr/local/src/hsds/hsds/ COPY hsds/util/*.py /usr/local/src/hsds/hsds/util/ COPY admin/config/config.yml /etc/hsds/ COPY admin/config/config.yml /usr/local/src/hsds/admin/config/ COPY entrypoint.sh / -RUN /bin/bash -c 'cd /usr/local/src/hsds; pip install -e ".[azure]" ; cd -' +RUN /bin/bash -c 'cd /usr/local/src/hsds; \ + pip install build;\ + python -m build;\ + pip install dist/hsds-0.8.2.tar.gz;\ + cd -' EXPOSE 5100-5999 ENTRYPOINT ["/bin/bash", "-c", "/entrypoint.sh"] diff --git a/README.md b/README.md index d7d4e8f6..14d2e798 100755 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ HSDS is a web service that implements a REST-based web service for HDF5 data stores. Data can be stored in either a POSIX files system, or using object-based storage such as AWS S3, Azure Blob Storage, or [MinIO](https://min.io). -HSDS can be run a single machine using Docker or on a cluster using Kubernetes (or AKS on Microsoft Azure). +HSDS can be run a single machine or on a cluster using Kubernetes (or AKS on Microsoft Azure). In addition, HSDS can be run in serverless mode with AWS Lambda or h5pyd local mode. diff --git a/build.sh b/build.sh index 191ae0c7..44bccf49 100755 --- a/build.sh +++ b/build.sh @@ -27,11 +27,17 @@ if [ $run_pyflakes ]; then fi fi -echo "running setup.py" -python setup.py install +pip install --upgrade build -echo "clean stopped containers" -docker rm -v $(docker ps -aq -f status=exited) +echo "running build" +python -m build +pip install dist/hsds-8.8.2.tar.gz -echo "building docker image" -docker build -t hdfgroup/hsds . +command -v docker +if [ $? -ne 1 ]; then + echo "clean stopped containers" + docker rm -v $(docker ps -aq -f status=exited) + + echo "building docker image" + docker build -t hdfgroup/hsds . +fi diff --git a/entrypoint.sh b/entrypoint.sh index ef8b123f..3850854f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -22,10 +22,6 @@ elif [ $NODE_TYPE == "head_node" ]; then echo "running hsds-headnode" export PYTHONUNBUFFERED="1" hsds-headnode -elif [ $NODE_TYPE == "rangeget" ]; then - echo "running hsds-rangeget" - export PYTHONUNBUFFERED="1" - hsds-rangeget else echo "Unknown NODE_TYPE: " $NODE_TYPE fi diff --git a/hsds/app.py b/hsds/app.py index 4394f567..8c8df5f0 100644 --- a/hsds/app.py +++ b/hsds/app.py @@ -19,7 +19,7 @@ from .hsds_app import HsdsApp from . import config -_HELP_USAGE = "Starts hsds a REST-based service for HDF5 data." +_HELP_USAGE = "Starts HSDS, a REST-based service for HDF5 data." _HELP_EPILOG = """Examples: diff --git a/hsds/basenode.py b/hsds/basenode.py index 3525c5a6..8dbd9799 100644 --- a/hsds/basenode.py +++ b/hsds/basenode.py @@ -33,7 +33,7 @@ from .util.k8sClient import getDnLabelSelector, getPodIps from . import hsds_logger as log -HSDS_VERSION = "0.8.1" +HSDS_VERSION = "0.8.2" def getVersion(): diff --git a/hsds/config.py b/hsds/config.py index 799f6036..bbbf4a95 100755 --- a/hsds/config.py +++ b/hsds/config.py @@ -86,7 +86,6 @@ def _load_cfg(): debug("set default location for config dirs") config_dirs = ["./", "/config", "/etc/hsds/"] # default locations for config_dir in config_dirs: - eprint("using config_dir:", config_dir) file_name = os.path.join(config_dir, "config.yml") debug("checking config path:", file_name) if os.path.isfile(file_name): diff --git a/setup.py b/setup.py deleted file mode 100644 index 2d2a748a..00000000 --- a/setup.py +++ /dev/null @@ -1,85 +0,0 @@ -from setuptools import setup - -# run: -# setup.py install -# or (if you'll be modifying the package): -# setup.py develop -# To use a consistent encoding -# To upload to PyPI: -# twine upload dist/* -# -# Tag the release in github! -# - -classifiers = [ - "Environment :: Console", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: English", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3", - "Topic :: Internet :: WWW/HTTP :: HTTP Servers", - "Topic :: Scientific/Engineering", -] - -# -# The specific versions given below are not desirable, -# but there are needed given aiobotocore and botocore -# requirements (e.g. see: https://github.com/boto/botocore/issues/2926#issuecomment-1535456104) -# as well as matching with packages present -# in the Docker build base image -# -install_requires = [ - "aiohttp==3.8.4", - "aiobotocore==2.5.0", - "aiohttp_cors", - "aiofiles", - "botocore", - "cryptography", - "numcodecs", - "numpy", - "psutil", - "pyjwt", - "pytz", - "pyyaml", - "requests-unixsocket", - "simplejson", -] - - -setup( - name="hsds", - version="0.8.1", - description="HDF REST API", - url="http://github.com/HDFGroup/hsds", - author="John Readey", - author_email="jreadey@hdfgrouup.org", - license="Apache", - packages=["hsds", "hsds.util", "admin"], - install_requires=install_requires, - setup_requires=["setuptools"], - extras_require={"azure": ["azure-storage-blob"]}, - zip_safe=False, - classifiers=classifiers, - include_package_data=True, - data_files=[ - ( - "admin", - [ - "admin/config/config.yml", - ], - ) - ], - entry_points={ - "console_scripts": [ - "hsds = hsds.app:main", - "hsds-datanode = hsds.datanode:main", - "hsds-servicenode = hsds.servicenode:main", - "hsds-headnode = hsds.headnode:main", - "hsds-rangeget = hsds.rangeget_proxy:main", - "hsds-node = hsds.node_runner:main", - "hsds-chunklocator = hsds.chunklocator:main" - ] - }, -) diff --git a/tests/integ/acl_test.py b/tests/integ/acl_test.py index ef0f943a..22833a3b 100644 --- a/tests/integ/acl_test.py +++ b/tests/integ/acl_test.py @@ -103,9 +103,12 @@ def testGetAcl(self): # try fetching an ACL from a user who doesn't have readACL permissions req = helper.getEndpoint() + "/acls/" + username user2name = config.get("user2_name") - headers = helper.getRequestHeaders(domain=self.base_domain, username=user2name) - rsp = self.session.get(req, headers=headers) - self.assertEqual(rsp.status_code, 403) # forbidden + if user2name: + headers = helper.getRequestHeaders(domain=self.base_domain, username=user2name) + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 403) # forbidden + else: + print("user2_name not set") def testGetAcls(self): print("testGetAcls", self.base_domain) @@ -224,9 +227,12 @@ def testGetAcls(self): # try fetching ACLs from a user who doesn't have readACL permissions req = helper.getEndpoint() + "/acls" user2name = config.get("user2_name") - headers = helper.getRequestHeaders(domain=self.base_domain, username=user2name) - rsp = self.session.get(req, headers=headers) - self.assertEqual(rsp.status_code, 403) # forbidden + if user2name: + headers = helper.getRequestHeaders(domain=self.base_domain, username=user2name) + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 403) # forbidden + else: + print("user2name not set") def testPutAcl(self): print("testPutAcl", self.base_domain) @@ -234,6 +240,9 @@ def testPutAcl(self): # create an ACL for "test_user2" with read and update access user2name = config.get("user2_name") + if not user2name: + print("user2_name not set") + return req = helper.getEndpoint() + "/acls/" + user2name perm = {"read": True, "update": True} diff --git a/tests/integ/attr_test.py b/tests/integ/attr_test.py index c8fc172c..072d25f4 100644 --- a/tests/integ/attr_test.py +++ b/tests/integ/attr_test.py @@ -14,6 +14,7 @@ import json import numpy as np import helper +import config class AttributeTest(unittest.TestCase): @@ -238,11 +239,15 @@ def testObjAttr(self): self.assertEqual(rsp.status_code, 404) # not found # try adding the attribute as a different user - headers = helper.getRequestHeaders( - domain=self.base_domain, username="test_user2" - ) - rsp = self.session.put(req, data=json.dumps(attr_payload), headers=headers) - self.assertEqual(rsp.status_code, 403) # forbidden + user2_name = config.get("user2_name") + if user2_name: + headers = helper.getRequestHeaders( + domain=self.base_domain, username="test_user2" + ) + rsp = self.session.put(req, data=json.dumps(attr_payload), headers=headers) + self.assertEqual(rsp.status_code, 403) # forbidden + else: + print("user2_name not set") # try adding again with original user, but outside this domain another_domain = helper.getParentDomain(self.base_domain) diff --git a/tests/integ/dataset_test.py b/tests/integ/dataset_test.py index ff50d138..0f6b930c 100755 --- a/tests/integ/dataset_test.py +++ b/tests/integ/dataset_test.py @@ -134,14 +134,18 @@ def testScalarDataset(self): # self.assertTrue("allocated_size" in rspJson) # try get with a different user (who has read permission) - headers = helper.getRequestHeaders(domain=domain, username="test_user2") - rsp = self.session.get(req, headers=headers) - if config.get("default_public"): - self.assertEqual(rsp.status_code, 200) - rspJson = json.loads(rsp.text) - self.assertEqual(rspJson["id"], dset_id) + user2_name = config.get('user2_name') + if user2_name: + headers = helper.getRequestHeaders(domain=domain, username=user2_name) + rsp = self.session.get(req, headers=headers) + if config.get("default_public"): + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["id"], dset_id) + else: + self.assertEqual(rsp.status_code, 403) else: - self.assertEqual(rsp.status_code, 403) + print('user2_name not set') # try to do a GET with a different domain (should fail) another_domain = self.base_domain + "/testScalarDataset2.h5" @@ -152,9 +156,12 @@ def testScalarDataset(self): self.assertEqual(rsp.status_code, 400) # try DELETE with user who doesn't have create permission on this domain - headers = helper.getRequestHeaders(domain=domain, username="test_user2") - rsp = self.session.delete(req, headers=headers) - self.assertEqual(rsp.status_code, 403) # forbidden + if user2_name: + headers = helper.getRequestHeaders(domain=domain, username=user2_name) + rsp = self.session.delete(req, headers=headers) + self.assertEqual(rsp.status_code, 403) # forbidden + else: + print("user2_name not set") # try to do a DELETE with a different domain (should fail) # Test creation/deletion of scalar dataset obj @@ -475,9 +482,13 @@ def testDelete(self): self.assertEqual(rspJson["id"], dset_id) # try DELETE with user who doesn't have create permission on this domain - headers = helper.getRequestHeaders(domain=domain, username="test_user2") - rsp = self.session.delete(req, headers=headers) - self.assertEqual(rsp.status_code, 403) # forbidden + user2_name = config.get('user2_name') + if user2_name: + headers = helper.getRequestHeaders(domain=domain, username=user2_name) + rsp = self.session.delete(req, headers=headers) + self.assertEqual(rsp.status_code, 403) # forbidden + else: + print("test_user2 not set") # try to do a DELETE with a different domain (should fail) another_domain = helper.getParentDomain(domain) diff --git a/tests/integ/datatype_test.py b/tests/integ/datatype_test.py index 788cdee9..deb576a4 100755 --- a/tests/integ/datatype_test.py +++ b/tests/integ/datatype_test.py @@ -72,16 +72,26 @@ def testCommittedType(self): self.assertEqual(type_json["base"], "H5T_IEEE_F32LE") # try get with a different user (who has read permission) - headers = helper.getRequestHeaders( - domain=self.base_domain, username="test_user2" - ) - rsp = self.session.get(req, headers=headers) - if config.get("default_public"): - self.assertEqual(rsp.status_code, 200) - rspJson = json.loads(rsp.text) - self.assertEqual(rspJson["root"], root_uuid) + test_user2 = config.get("user2_name") # some tests will be skipped if not + if test_user2: + headers = helper.getRequestHeaders( + domain=self.base_domain, username="test_user2" + ) + rsp = self.session.get(req, headers=headers) + if config.get("default_public"): + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["root"], root_uuid) + else: + self.assertEqual(rsp.status_code, 403) + # try DELETE with user who doesn't have create permission on this domain + headers = helper.getRequestHeaders( + domain=self.base_domain, username="test_user2" + ) + rsp = self.session.delete(req, headers=headers) + self.assertEqual(rsp.status_code, 403) # forbidden else: - self.assertEqual(rsp.status_code, 403) + print('test_user2 not set') # try to do a GET with a different domain (should fail) another_domain = helper.getParentDomain(self.base_domain) @@ -89,13 +99,6 @@ def testCommittedType(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 400) - # try DELETE with user who doesn't have create permission on this domain - headers = helper.getRequestHeaders( - domain=self.base_domain, username="test_user2" - ) - rsp = self.session.delete(req, headers=headers) - self.assertEqual(rsp.status_code, 403) # forbidden - # try to do a DELETE with a different domain (should fail) another_domain = helper.getParentDomain(self.base_domain) headers = helper.getRequestHeaders(domain=another_domain) diff --git a/tests/integ/domain_test.py b/tests/integ/domain_test.py index cbaeac3d..c0e22feb 100755 --- a/tests/integ/domain_test.py +++ b/tests/integ/domain_test.py @@ -281,6 +281,7 @@ def testGetTopLevelDomain(self): domain = "/home" print("testGetTopLevelDomain", domain) headers = helper.getRequestHeaders(domain=domain) + user_name = config.get("user_name") req = helper.getEndpoint() + "/" rsp = self.session.get(req, headers=headers) @@ -291,7 +292,7 @@ def testGetTopLevelDomain(self): self.assertTrue("hrefs" in rspJson) self.assertTrue("class" in rspJson) self.assertEqual(rspJson["class"], "folder") - domain = "test_user1.home" + domain = f"{user_name}.home" headers = helper.getRequestHeaders(domain=domain) req = helper.getEndpoint() + "/" @@ -564,6 +565,11 @@ def testCreateFolder(self): def testAclInheritence(self): # this test is here (rather than acl_test.py) since we need to create domains in a folder print("testAclInheritence", self.base_domain) + user2name = config.get("user2_name") + if not user2name: + print("user2_name not set, skipping test") + return + folder = self.base_domain + "/a_folder" headers = helper.getRequestHeaders(domain=folder) req = helper.getEndpoint() + "/" @@ -573,7 +579,6 @@ def testAclInheritence(self): default_public = config.get("default_public") # create an ACL for "test_user2" with read and update access - user2name = config.get("user2_name") req = helper.getEndpoint() + "/acls/" + user2name perm = {"read": True, "update": True} @@ -809,9 +814,12 @@ def testDeleteDomain(self): # try deleting the domain with a user who doesn't have permissions' user2_name = config.get("user2_name") - headers = helper.getRequestHeaders(domain=self.base_domain, username=user2_name) - rsp = self.session.delete(req, headers=headers) - self.assertEqual(rsp.status_code, 403) # forbidden + if user2_name: + headers = helper.getRequestHeaders(domain=self.base_domain, username=user2_name) + rsp = self.session.delete(req, headers=headers) + self.assertEqual(rsp.status_code, 403) # forbidden + else: + print("user2_name not set") # delete the domain (with the orginal user) headers = helper.getRequestHeaders(domain=domain) @@ -849,27 +857,35 @@ def testDeleteDomain(self): msg += "environment variables to enable" print(msg) - # try creating a folder using the owner flag - try: - admin_username = config.get("admin_username") - admin_passwd = config.get("admin_password") - username = config.get("user2_name") - new_domain = f"{self.base_domain}/{username}_folder" - body = {"folder": True, "owner": username} - headers = helper.getRequestHeaders( - domain=new_domain, username=admin_username, password=admin_passwd - ) - rsp = self.session.put(req, headers=headers, data=json.dumps(body)) - self.assertEqual(rsp.status_code, 201) - - headers = helper.getRequestHeaders(domain=new_domain, username=username) - rsp = self.session.get(req, headers=headers) - self.assertEqual(rsp.status_code, 200) - rspJson = json.loads(rsp.text) - except KeyError: - msg = "Skipping domain create with owner test, set ADMIN_USERNAME" - msg += " and ADMIN_PASSWORD environment variables to enable" - print(msg) + username = config.get("user2_name") + admin_username = config.get("admin_username") + + if username and admin_username: + + # try creating a folder using the owner flag + try: + admin_passwd = config.get("admin_password") + new_domain = f"{self.base_domain}/{username}_folder" + body = {"folder": True, "owner": username} + headers = helper.getRequestHeaders( + domain=new_domain, username=admin_username, password=admin_passwd + ) + rsp = self.session.put(req, headers=headers, data=json.dumps(body)) + self.assertEqual(rsp.status_code, 201) + + headers = helper.getRequestHeaders(domain=new_domain, username=username) + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + except KeyError: + msg = "Skipping domain create with owner test, set ADMIN_USERNAME" + msg += " and ADMIN_PASSWORD environment variables to enable" + print(msg) + else: + if not username: + print("user2_name not set") + elif not admin_username: + print("admin_username not set") def testDomainCollections(self): domain = helper.getTestDomain("tall.h5") diff --git a/tests/integ/group_test.py b/tests/integ/group_test.py index 31b31aa3..eebd1b02 100755 --- a/tests/integ/group_test.py +++ b/tests/integ/group_test.py @@ -60,16 +60,20 @@ def testGetRootGroup(self): self.assertTrue("attributeCount" in rspJson) # try get with a different user (who has read permission) - headers = helper.getRequestHeaders( - domain=self.base_domain, username="test_user2" - ) - rsp = self.session.get(req, headers=headers) - if config.get("default_public"): - self.assertEqual(rsp.status_code, 200) - rspJson = json.loads(rsp.text) - self.assertEqual(rspJson["root"], root_uuid) + user2_name = config.get("user2_name") + if user2_name: + headers = helper.getRequestHeaders( + domain=self.base_domain, username=user2_name + ) + rsp = self.session.get(req, headers=headers) + if config.get("default_public"): + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertEqual(rspJson["root"], root_uuid) + else: + self.assertEqual(rsp.status_code, 403) else: - self.assertEqual(rsp.status_code, 403) + print("user2_name not set") # try to do a GET with a different domain (should fail) another_domain = helper.getParentDomain(self.base_domain) @@ -217,6 +221,11 @@ def testPost(self): self.assertEqual(rspJson["alias"], []) # try POST with user who doesn't have create permission on this domain + test_user2 = config.get("user2_name") # some tests will be skipped if not set + if not test_user2: + print("test_user2 not set") + return + headers = helper.getRequestHeaders( domain=self.base_domain, username="test_user2" ) @@ -385,11 +394,15 @@ def testDelete(self): # self.assertEqual(rspJson["domain"], self.base_domain) #TBD # try DELETE with user who doesn't have create permission on this domain - headers = helper.getRequestHeaders( - domain=self.base_domain, username="test_user2" - ) - rsp = self.session.delete(req, headers=headers) - self.assertEqual(rsp.status_code, 403) # forbidden + test_user2 = config.get("user2_name") # some tests will be skipped if not set + if test_user2: + headers = helper.getRequestHeaders( + domain=self.base_domain, username="test_user2" + ) + rsp = self.session.delete(req, headers=headers) + self.assertEqual(rsp.status_code, 403) # forbidden + else: + print("test_user2 not set") # try to do a DELETE with a different domain (should fail) another_domain = helper.getParentDomain(self.base_domain) diff --git a/tests/integ/link_test.py b/tests/integ/link_test.py index 7bd1de64..02392cdb 100755 --- a/tests/integ/link_test.py +++ b/tests/integ/link_test.py @@ -15,6 +15,7 @@ import json import uuid import helper +import config class LinkTest(unittest.TestCase): @@ -36,6 +37,7 @@ def testHardLink(self): helper.setupDomain(domain) headers = helper.getRequestHeaders(domain=domain) req = helper.getEndpoint() + "/" + test_user2 = config.get("user2_name") # some tests will be skipped if not set rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) @@ -66,12 +68,16 @@ def testHardLink(self): self.assertEqual(rsp.status_code, 404) # link doesn't exist yet # try creating a link with a different user (should fail) - headers = helper.getRequestHeaders(domain=domain, username="test_user2") - payload = {"id": grp1_id} - rsp = self.session.put(req, data=json.dumps(payload), headers=headers) - self.assertEqual(rsp.status_code, 403) # forbidden + if test_user2: + headers = helper.getRequestHeaders(domain=domain, username=test_user2) + payload = {"id": grp1_id} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 403) # forbidden + else: + print("test_user2 name not set") # create "/g1" with original user + payload = {"id": grp1_id} headers = helper.getRequestHeaders(domain=domain) rsp = self.session.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # created @@ -107,10 +113,13 @@ def testHardLink(self): self.assertEqual(rspJson["linkCount"], 1) # link count is 1 # try deleting link with a different user (should fail) - headers = helper.getRequestHeaders(domain=domain, username="test_user2") - req = helper.getEndpoint() + "/groups/" + root_id + "/links/" + link_title - rsp = self.session.delete(req, headers=headers) - self.assertEqual(rsp.status_code, 403) # forbidden + if test_user2: + headers = helper.getRequestHeaders(domain=domain, username=test_user2) + req = helper.getEndpoint() + "/groups/" + root_id + "/links/" + link_title + rsp = self.session.delete(req, headers=headers) + self.assertEqual(rsp.status_code, 403) # forbidden + else: + print("user2_name not set") # delete the link with original user req = helper.getEndpoint() + "/groups/" + root_id + "/links/" + link_title