Update scripts to use docker compose v2 (#340)

.github/workflows/python-package.yml

@@ -8,14 +8,17 @@ on:
   pull_request:
     branches: [master]
 
+env:
+  ROOT_DIR: ${{github.workspace}}/data
+
 jobs:
   build-and-test:
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-        test-other: ["none", "h5pyd"]
+        build-method: ["manual", "docker"]
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -49,9 +52,9 @@ jobs:
         shell: bash
         run: |
           pytest
-
-      - name: Run integration tests
+      - name: Start HSDS (Manual)
         shell: bash
+        if: ${{matrix.build-method == 'manual'}}
         env:
           ADMIN_PASSWORD: admin
           ADMIN_USERNAME: admin
@@ -60,52 +63,94 @@ jobs:
           USER2_NAME: test_user2
           USER2_PASSWORD: test
           HSDS_ENDPOINT: http://127.0.0.1:5101
-
+          BUCKET_NAME: hsdstest
         run: |
           mkdir data
           mkdir data/hsdstest
           cp admin/config/groups.default admin/config/groups.txt
           cp admin/config/passwd.default admin/config/passwd.txt
           hsds --root_dir data --host localhost --port 5101 --password_file admin/config/passwd.txt --logfile hs.log   --loglevel DEBUG  --config_dir=admin/config --count=4 &
-          sleep 10 # let the nodes get ready
+
+      - name: Start HSDS (Docker)
+        shell: bash
+        if: ${{matrix.build-method == 'docker' && matrix.os != 'windows-latest'}}
+        env:
+          ADMIN_PASSWORD: admin
+          ADMIN_USERNAME: admin
+          USER_NAME: test_user1
+          USER_PASSWORD: test
+          USER2_NAME: test_user2
+          USER2_PASSWORD: test
+          HSDS_ENDPOINT: http://127.0.0.1:5101
+          BUCKET_NAME: hsdstest
+        run: |
+          mkdir data
+          mkdir data/hsdstest
+          cp admin/config/groups.default admin/config/groups.txt
+          cp admin/config/passwd.default admin/config/passwd.txt
+          ./build.sh --nolint
+          ./runall.sh
+
+      - name: Wait for node startup
+        shell: bash
+        run: |
+          sleep 45
+
+      - name: Run HSDS tests
+        if: ${{!(matrix.build-method == 'docker' && matrix.os == 'windows-latest')}}
+        shell: bash
+        env:
+          ADMIN_PASSWORD: admin
+          ADMIN_USERNAME: admin
+          USER_NAME: test_user1
+          USER_PASSWORD: test
+          USER2_NAME: test_user2
+          USER2_PASSWORD: test
+          BUCKET_NAME: hsdstest
+        run: |
           pytest tests/integ/setup_test.py
           pytest tests/integ
 
       - name: Checkout h5pyd
-        if: ${{ ( matrix.test-other ==  'h5pyd' ) && ( matrix.os != 'windows-latest' ) }}
+        if: ${{ ( matrix.os != 'windows-latest' ) }}
         uses: actions/checkout@v4
         with:
           repository: HDFGroup/h5pyd
           path: ${{github.workspace}}/h5pyd
 
       - name: Install h5pyd
-        if: ${{ ( matrix.test-other ==  'h5pyd' ) && ( matrix.os != 'windows-latest' ) }}
+        if: ${{ ( matrix.os != 'windows-latest' ) }}
         working-directory: ${{github.workspace}}/h5pyd
         run: |
           pip install . -v
         
       - name: Create h5pyd test folder
-        if: ${{ ( matrix.test-other ==  'h5pyd' ) && ( matrix.os != 'windows-latest' ) }}
+        if: ${{ ( matrix.os != 'windows-latest' ) }}
         working-directory: ${{github.workspace}}/h5pyd
         env:
           HS_USERNAME: test_user1
           HS_PASSWORD: test
           TEST2_USERNAME: test_user1
           TEST2_PASSWORD: test
-          HS_ENDPOINT: http://127.0.0.1:5101
           H5PYD_TEST_FOLDER: /home/test_user1/h5pyd_tests/
+          HS_ENDPOINT: http://127.0.0.1:5101
         run: |
-          hstouch -e http://127.0.0.1:5101 /home/test_user1/h5pyd_tests/
+          hstouch -e $HS_ENDPOINT /home/test_user1/h5pyd_tests/
 
       - name: Run h5pyd tests
-        if: ${{ ( matrix.test-other ==  'h5pyd' ) && ( matrix.os != 'windows-latest' ) }}
+        if: ${{ ( matrix.os != 'windows-latest' ) }}
         working-directory: ${{github.workspace}}/h5pyd
         env:
           HS_USERNAME: test_user1
           HS_PASSWORD: test
           TEST2_USERNAME: test_user1
           TEST2_PASSWORD: test
-          HS_ENDPOINT: http://127.0.0.1:5101
           H5PYD_TEST_FOLDER: /home/test_user1/h5pyd_tests/
+          HS_ENDPOINT: http://127.0.0.1:5101
         run: |
           python testall.py
+
+      - name: Shut down Docker
+        if: ${{matrix.build-method == 'docker' && matrix.os != 'windows-latest'}}
+        run: |
+          ./stopall.sh

docs/design/azure/azure.md

@@ -28,7 +28,7 @@ The current implementation of HSDS uses these AWS specific technologies:
 - AWS S3 - Storage of HDF data
 - AWS DynamoDB - HSDS usernames and passwords
 
-When running HSDS on a single VM using Docker, there should be no differences between AWS and Azure as far as how containers are set (i.e. the same docker-compose configuration should work for both).  
+When running HSDS on a single VM using Docker, there should be no differences between AWS and Azure as far as how containers are set (i.e. the same docker compose configuration should work for both).
 
 For running HSDS in a self-managed Kubernetes cluster, this should also work the same way on Azure.  In addition, both Amazon and Microsoft offers a supported Kubernetes services: AWS EKS and Azure AKS.  But self-hosted Kubernetes, vs AWS EKS vs. Azure should have minimal differences given common Kubernetes API and command line tools (e.g. kubectl), so we won't address Kubernetes differences here.
 

docs/docker_install_aws.md

@@ -4,7 +4,7 @@ Installation using Docker with AWS S3 storage
 Export environment variables as shown in "Sample .bashrc" below.
 
 1. Install Python 3 (e.g. with Miniconda <https://docs.conda.io/en/latest/miniconda.html>)
-2. Install Docker and docker-compose if necessary. See [Docker Setup](setup_docker.md) 
+2. Install Docker and docker compose if necessary. See [Docker Setup](setup_docker.md)
 3. Create a bucket for HSDS, using aws cli tools or aws management console
 4. Get project source code: `$ git clone https://github.com/HDFGroup/hsds`
 5. Build the docker image: `$ ./build.sh --nolint`

docs/docker_install_azure.md

@@ -68,7 +68,7 @@ These environment variables will be passed to the Docker containers on startup.
 Follow the following steps to setup HSDS:
 
 1. SSH to the VM created above.  Replace [publicIpAddress] with the public IP displayed in the output of your VM creation command above: `ssh $VM_USER@[publicIpAddress]`
-2. Install Docker and docker-compose if necessary.   See [Docker Setup](setup_docker.md)
+2. Install Docker and docker compose if necessary.   See [Docker Setup](setup_docker.md)
 3. Get project source code: `git clone https://github.com/HDFGroup/hsds`
 4. If you plan to use HTTP Basic Auth (usernames and passwords managed by the service), go to hsds/admin/config directory: `cd admin/config`, and copy the file "passwd.default" to "passwd.txt".  Add any usernames/passwords you wish.  Modify existing passwords (for admin, test_user1, test_user2, etc.) for security.  If you wish to use Azure Active Directory for authentication, follow the instructions in [Azure Active Directory Setup](azure_ad_setup.md)
 5. If group-level permissions are desired (See [Authorization](authorization.md)), copy the file "groups.default" to "groups.txt".  Modify existing groups as needed

docs/docker_install_posix.md

@@ -9,7 +9,7 @@ Export environment variables as shown in "Sample .bashrc" below.
 2. Create a directory for storage files and set the ROOT_DIR environment variable to point to it
 3. Create a subdirectory under ROOT_DIR that will be the default location when "bucket" is not defined.
 4. Set the environment variable BUCKET_NAME to the name of the subdirectory
-5. Install Docker and docker-compose if necessary.  See [Docker Setup](setup_docker.md)
+5. Install Docker and docker compose if necessary.  See [Docker Setup](setup_docker.md)
 6. Get project source code: `$ git clone https://github.com/HDFGroup/hsds`
 7. Go to admin/config directory: `$ cd hsds/admin/config`
 8. Copy the file "passwd.default" to "passwd.txt".  Add any usernames/passwords you wish.  Modify existing passwords (for admin, test_user1, test_user2) for security
@@ -53,5 +53,5 @@ To get the latest codes changes from the HSDS repo do the following:
 
 1. Shutdown the service: `$ stopall.sh`
 2. Get code changes: `$ git pull`
-3. Build a new Docker image: `$ docker-compose build`
+3. Build a new Docker image: `$ docker compose build`
 4. Start the service: `$ ./runall.sh`

docs/docker_install_tencent.md

@@ -13,7 +13,7 @@ values should be fine.  Record the bucket name and endpoint value.
 8. For the security group, enable ports 22, and 80
 9. Start instance and ssh into the instance.  The following steps will be performed on the instance
 10. Install Python 3 (e.g. with Miniconda <https://docs.conda.io/en/latest/miniconda.html>)
-11. Install docker-compose by running: `$ curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose`, and `$ chmod +x /usr/local/bin/docker-compose` 
+11. Install docker compose by running: `$ curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker compose`, and `$ chmod +x /usr/local/bin/docker-compose`
 12. If docker is not running, start it with: `systemctl start docker`
 13. Get project source code: `$ git clone https://github.com/HDFGroup/hsds`
 14. Go to the hsds directory and run: `$ ./build.sh`

docs/setup_docker.md

@@ -16,7 +16,7 @@ Run the following commands to install Docker on Linux/Ubuntu:
 7. Log out and back in again (you may also need to stop/start docker service)
 8. `docker ps` to verify that Docker is running.
 
-Install docker-compose:
+Install docker compose:
 
 1. See: <https://docs.docker.com/compose/install/>
 

runall.sh

@@ -132,8 +132,8 @@ if [[ ${NO_DOCKER} ]]; then
   echo "using password file: ${PASSWORD_FILE}"
 
 else
-    # check that docker-compose is available
-    docker-compose --version >/dev/null || exit 1
+    # check that docker compose is available
+    docker compose --version >/dev/null || exit 1
     if [[ -z ${COMPOSE_PROJECT_NAME} ]]; then
       export COMPOSE_PROJECT_NAME=hsds  # use "hsds_" as prefix for container names
     fi
@@ -201,12 +201,12 @@ if [[ $NO_DOCKER ]] ; then
 else
   if [[ $DOCKER_CMD == "down" ]]; then
     # use the compose file to shutdown the sevice
-    echo "Running docker-compose -f ${COMPOSE_FILE} down"
-    docker-compose -f ${COMPOSE_FILE} down
+    echo "Running docker compose -f ${COMPOSE_FILE} down"
+    docker compose -f ${COMPOSE_FILE} down
     exit 0  # can quit now
   else
-    echo "Running docker-compose -f ${COMPOSE_FILE} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES}"
-    docker-compose -f ${COMPOSE_FILE} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES}
+    echo "Running docker compose -f ${COMPOSE_FILE} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES}"
+    docker compose -f ${COMPOSE_FILE} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES}
   fi
 
   # wait for the server to be ready

stopall.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
 #
-# Shutdown HSDS with "docker-compose down" using the appropriate compose file
+# Shutdown HSDS with "docker compose down" using the appropriate compose file
 #
 ./runall.sh --stop

tests/perf/write/start.sh

@@ -27,4 +27,4 @@ fi
 
 echo "count $count"
 
-docker-compose -f docker-compose.yml up -d --scale hswritetest=$count
+docker compose -f docker-compose.yml up -d --scale hswritetest=$count

tests/perf/write/stop.sh

@@ -1 +1 @@
-docker-compose -f docker-compose.yml down 
+docker compose -f docker-compose.yml down