Add codespace configuration for h5pyd (#187)

* added codespace config

* added requirements.txt

* revised examples

* move requirements.txt to .devcontainer

* support s3fs for anonymous creds

* set hsds loglevel to ERROR

* updated notebook environment

* updated notebook markdown

* remove extraneious .domain.json file

.devcontainer/devcontainer.json

@@ -0,0 +1,29 @@
+{
+  "name": "h5pyd with HSDS",
+  "hostRequirements": {
+    "cpus": 4
+  },
+  "dockerComposeFile": "docker-compose.yaml",
+  "updateContentCommand": "python3 -m pip install -r .devcontainer/requirements.txt",
+  "postCreateCommand": "",
+  "service": "app",
+	"workspaceFolder": "/workspace",
+	"forwardPorts": [5101],
+	"portsAttributes": {
+		"5101": {"label": "HSDS port", "onAutoForward": "silent"}
+	},
+  "features": {
+    "ghcr.io/devcontainers/features/docker-outside-of-docker": {}
+  },
+  "customizations": {
+    "codespaces": {
+      "openFiles": []
+    },
+    "vscode": {
+      "extensions": [
+          "ms-python.python",
+          "ms-toolsai.jupyter"
+          ]
+    }
+  }
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,107 @@
+version: "3"
+services:
+  app:
+    image: "mcr.microsoft.com/devcontainers/universal:2"
+    environment:
+      - HS_ENDPOINT=http://localhost:5101
+      - HS_USERNAME=test_user1
+      - HS_PASSWORD=test
+      - H5PYD_TEST_FOLDER=/home/test_user1/h5pyd_test/
+    volumes:
+      - ..:/workspace:cached
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+
+    # Runs app on the same network as the SN container, allows "forwardPorts" in devcontainer.json function.
+    network_mode: service:sn
+  head:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 512m
+    environment:
+      - TARGET_SN_COUNT=1
+      - TARGET_DN_COUNT=4
+      - NODE_TYPE=head_node
+    ports:
+      - 5100
+    volumes:
+      - ../.hsds/data:/data
+      - ../.hsds/config/:/config/
+  dn1:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 1g
+    environment:
+      - NODE_TYPE=dn
+      - ROOT_DIR=/data
+    ports:
+      - 6101
+    depends_on:
+      - head
+    volumes:
+      - ../.hsds/data:/data
+      - ../.hsds/config/:/config/
+    links:
+      - head
+  dn2:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 1g
+    environment:
+      - NODE_TYPE=dn
+    ports:
+      - 6102
+    depends_on:
+      - head
+    volumes:
+      - ../.hsds/data:/data
+      - ../.hsds/config/:/config/
+    links:
+      - head
+  dn3:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 1g
+    environment:
+      - NODE_TYPE=dn
+    ports:
+      - 6103
+    depends_on:
+      - head
+    volumes:
+      - ../.hsds/data:/data
+      - ../.hsds/config/:/config/
+    links:
+      - head
+  dn4:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 1g
+    environment:
+      - NODE_TYPE=dn
+    ports:
+      - 6104
+    depends_on:
+      - head
+    volumes:
+      - ../.hsds/data:/data
+      - ../.hsds/config/:/config/
+    links:
+      - head
+  sn:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 1g
+    environment:
+      - SN_PORT=5101
+      - NODE_TYPE=sn
+    ports:
+      - 5101:5101
+    depends_on:
+      - head
+    volumes:
+      - ../.hsds/data:/data
+      - ../.hsds/config/:/config/
+    links:
+      - head

.devcontainer/requirements.txt

@@ -0,0 +1,40 @@
+aiobotocore==2.12.3
+aiohttp==3.9.5
+aioitertools==0.11.0
+aiosignal==1.3.1
+attrs==23.2.0
+botocore==1.34.69
+build==1.2.1
+certifi==2024.2.2
+charset-normalizer==3.3.2
+contourpy==1.2.1
+cycler==0.12.1
+flake8==7.0.0
+fonttools==4.51.0
+frozenlist==1.4.1
+fsspec==2024.3.1
+h5py==3.11.0
+h5pyd @ git+https://github.com/hdfgroup/h5pyd
+idna==3.7
+jmespath==1.0.1
+kiwisolver==1.4.5
+matplotlib==3.8.4
+mccabe==0.7.0
+multidict==6.0.5
+numpy==1.26.4
+packaging==24.0
+pillow==10.3.0
+pycodestyle==2.11.1
+pyflakes==3.2.0
+PyJWT==2.8.0
+pyparsing==3.1.2
+pyproject_hooks==1.0.0
+python-dateutil==2.9.0.post0
+pytz==2024.1
+requests==2.31.0
+requests-unixsocket==0.3.0
+s3fs==2024.3.1
+six==1.16.0
+urllib3==2.2.1
+wrapt==1.16.0
+yarl==1.9.4

.gitignore

@@ -66,3 +66,6 @@ target/
 
 # macOS stuff
 .DS_Store
+
+# hsds data
+data/

.hsds/config/override.yml

@@ -0,0 +1,9 @@
+log_level: ERROR 
+bucket_name: hsdstest
+server_name: "HSDS for Github codespaces"
+aws_region: us-west-2 # (original was us-east-1)
+aws_s3_gateway: http://s3.us-west-2.amazonaws.com/ # (original was null)
+aws_s3_no_sign_request: True # (original was false)
+root_dir: /data
+
+

.hsds/config/passwd.txt

@@ -0,0 +1,13 @@
+# HSDS password file template
+#
+#
+# This file contains a list of usernames/passwords that will be used to authenticate
+# requests to HSDS.
+# If using HTTP Basic Auth, copy file to "passwd.txt" in the same directory before deploying HSDS.
+# Otherwise, if using Azure Active Directory or Kerberos, don't copy this file - usernames will be
+# authenticated using those identity providers.
+# For production use, replace the "test" password below with secret passwords and add
+# and any new accounts desired.
+admin:admin
+test_user1:test
+test_user2:test

.hsds/data/hsdstest/home/.domain.json

@@ -0,0 +1 @@
+{"owner": "admin", "acls": {"admin": {"create": true, "read": true, "update": true, "delete": true, "readACL": true, "updateACL": true}, "default": {"create": false, "read": true, "update": false, "delete": false, "readACL": false, "updateACL": false}}, "created": 1708897646.0599918, "lastModified": 1708897646.0599918}

.hsds/data/hsdstest/home/test_user1/.domain.json

@@ -0,0 +1 @@
+{"owner": "test_user1", "acls": {"test_user1": {"create": true, "read": true, "update": true, "delete": true, "readACL": true, "updateACL": true}, "default": {"create": false, "read": true, "update": false, "delete": false, "readACL": false, "updateACL": false}}, "created": 1711992550.3733413, "lastModified": 1711992550.3733413}

.hsds/data/hsdstest/home/test_user1/h5pyd_test/.domain.json

@@ -0,0 +1 @@
+{"owner": "test_user1", "acls": {"test_user1": {"create": true, "read": true, "update": true, "delete": true, "readACL": true, "updateACL": true}, "default": {"create": false, "read": true, "update": false, "delete": false, "readACL": false, "updateACL": false}}, "created": 1711992550.3733413, "lastModified": 1711992550.3733413}

README.rst

@@ -1,16 +1,18 @@
+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/hdfgroup/h5pyd)
+
 h5pyd
 =====
 
 .. image:: https://travis-ci.org/HDFGroup/h5pyd.svg?branch=master
     :target: https://travis-ci.org/HDFGroup/h5pyd
 
-Python client library for HDF5 REST interface
+Python client library for HSDS
 
 
 Introduction
 ------------
-This repository contains library, test, and examples of h5pyd - a Python package for the
-HDF REST interface.
+This repository contains library, test, and examples of h5pyd - a Python package for HSDS 
+(Highly Scalable Data Service), or other HDF REST compatible web services.
 
 The h5pyd library provides a high-level interface to the REST specification that is generally
 easier to use than invoking http calls directly.
@@ -31,7 +33,6 @@ Related Projects
 ----------------
 
 * HSDS: https://github.com/HDFGroup/hsds
-* HDF Server: https://github.com/HDFGroup/h5serv
 * HDF5-JSON: https://github.com/HDFGroup/hdf5-json
 * h5py: https://github.com/h5py/h5py
 * REST API Documentation: https://github.com/HDFGroup/hdf-rest-api
@@ -50,10 +51,12 @@ From a release tarball or Git checkout::
 By default the examples look for a local instance of HSDS.  See the  https://github.com/HDFGroup/hsds
 for instructions on installing and running HSDS.
 
-These tests are also to designed to work with HSDS (see https://github.com/HDFGroup/hsds).  Install HSDS locally, or set environment variables (see next section)
-to point to an existing HSDS instance.
+These tests are also to designed to work with HSDS (see https://github.com/HDFGroup/hsds).  
+Install HSDS locally, or set environment variables (see next section)
+to point to an existing HSDS instance.  Alternatively, if you create as a Github codespace 
+(sell link above), HSDS will be automatically setup as part of the codespace.
 
-h5pyd can all be run in serverless mode with either AWS Lambda or direct mode (storage system accessed directly).
+Alternatively, h5pyd can all be run in serverless mode with either AWS Lambda or direct mode (storage system accessed directly).
 
 To use with AWS Lambda, set the HS_ENDPOINT to: "http+lambda://hslambda" where "hslambda" is the name
 of the lambda function.  When using AWS Lambda some additional environment variables need to be set:

environment.yml

@@ -1,15 +1,15 @@
-name: hsds
+name: h5pyd tutorial
 channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.6
+  - python=3.11
   - h5py
   - jupyter
+  - matplotlib
   - nb_conda_kernels
   - pytz
   - requests
-  - matplotlib
+  - s3fs
   - pip:
-      - h5pyd  #latest release on PyPi
-#      - git+https://github.com/HDFGroup/h5pyd.git #latest from master
+      - h5pyd @ git+https://github.com/hdfgroup/h5pyd  # latest from master