update compose files

bgruening · Oct 29, 2024 · 86dce83 · 86dce83
1 parent b2c02d6
commit 86dce83
Show file tree

Hide file tree

Showing 26 changed files with 177 additions and 133 deletions.
diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml
@@ -133,69 +133,43 @@ jobs:
         infrastructure:
           - name: galaxy-base
             files: -f docker-compose.yml
-            exclude_test:
-              - workflow_example1
           - name: galaxy-proxy-prefix
             files: -f docker-compose.yml
-            env: GALAXY_PROXY_PREFIX=/arbitrary_Galaxy-prefix GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost/arbitrary_Galaxy-prefix
+            env: GALAXY_PROXY_PREFIX=/arbitrary_Galaxy-prefix GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost/arbitrary_Galaxy-prefix EXTRA_SKIP_TESTS_BIOBLEND="not test_import_export_workflow_dict and not test_import_export_workflow_from_local_path"
             exclude_test:
-              - workflow_example1
+              - selenium
           - name: galaxy-htcondor
             files: -f docker-compose.yml -f docker-compose.htcondor.yml
-            exclude_test:
-              - workflow_example1
           - name: galaxy-slurm
             files: -f docker-compose.yml -f docker-compose.slurm.yml
             env: SLURM_NODE_COUNT=3
             options: --scale slurm_node=3
-            exclude_test:
-              - workflow_example1
           - name: galaxy-pulsar
             files: -f docker-compose.yml -f docker-compose.pulsar.yml
             exclude_test:
-              - workflow_example1
-              - workflow_mapping_by_sequencing
+              - workflow_quality_control
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job"
           - name: galaxy-pulsar-mq
             files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml
             exclude_test:
-              - workflow_example1
-              - workflow_mapping_by_sequencing
+              - workflow_quality_control
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job"
           - name: galaxy-k8s
             files: -f docker-compose.yml -f docker-compose.k8s.yml
-            exclude_test:
-              - bioblend
-              - workflow_example1
-              - workflow_ard
-              - workflow_mapping_by_sequencing
-              - selenium
           - name: galaxy-singularity
             files: -f docker-compose.yml -f docker-compose.singularity.yml
-            exclude_test:
-              - bioblend
-              - workflow_ard
-              - workflow_mapping_by_sequencing
-              - selenium
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver"
           - name: galaxy-pulsar-mq-singularity
             files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml -f docker-compose.singularity.yml
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job and not test_get_container_resolvers and not test_show_container_resolver"
             exclude_test:
-              - bioblend
-              - workflow_ard
-              - workflow_mapping_by_sequencing
-              - selenium
+              - workflow_quality_control
           - name: galaxy-slurm-singularity
             files: -f docker-compose.yml -f docker-compose.slurm.yml -f docker-compose.singularity.yml
-            exclude_test:
-              - bioblend
-              - workflow_ard
-              - workflow_mapping_by_sequencing
-              - selenium
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver"
           - name: galaxy-htcondor-singularity
             files: -f docker-compose.yml -f docker-compose.htcondor.yml -f docker-compose.singularity.yml
-            exclude_test:
-              - bioblend
-              - workflow_ard
-              - workflow_mapping_by_sequencing
-              - selenium
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver"
         test:
           - name: bioblend
             files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.bioblend.yml
@@ -208,10 +182,10 @@ jobs:
             workflow: sklearn/ard/ard.ga
             timeout: 60
             second_run: "true"
-          - name: workflow_mapping_by_sequencing
+          - name: workflow_quality_control
             files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
             exit-from: galaxy-workflow-test
-            workflow: training/variant-analysis/mapping-by-sequencing/mapping_by_sequencing.ga
+            workflow: training/sequence-analysis/quality-control/quality_control.ga
             timeout: 60
           - name: workflow_example1
             files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
@@ -273,6 +247,10 @@ jobs:
         shell: bash
         working-directory: ./compose
         continue-on-error: false
+      - name: Fix file names before saving artifacts
+        if: failure()
+        run: |
+          sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \;
       - name: Allow upload-artifact read access
         if: failure()
         run: sudo chmod -R +r ./compose/export/galaxy/database
@@ -289,7 +267,7 @@ jobs:
           sudo rm -rf export/galaxy/database
         working-directory: ./compose
       - name: Run tests a second time
-        if: matrix.test.second_run == 'true' && steps.run_check.run
+        if: matrix.test.second_run == 'true' && steps.run_check.outputs.run
         run: |
           export DOCKER_REGISTRY=${{ secrets.docker_registry }}
           export DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }}
@@ -315,6 +293,10 @@ jobs:
         shell: bash
         working-directory: ./compose
         continue-on-error: false
+      - name: Fix file names before saving artifacts
+        if: failure() && matrix.test.second_run == 'true'
+        run: |
+          sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \;
       - name: Allow upload-artifact read access
         if: failure() && matrix.test.second_run == 'true'
         run: sudo chmod -R +r ./compose/export/galaxy/database

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -6,10 +6,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-      with:
-        sparse-checkout: |
-          .github
-          compose
+    # - name: Cleanup to only use compose
+    #   run: rm -R docs galaxy test
     - name: Run shellcheck with reviewdog
       uses: reviewdog/[email protected]
       with:

diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
@@ -8,71 +8,43 @@ jobs:
         infrastructure:
           - name: galaxy-base
             files: -f docker-compose.yml
-            exclude_test:
-              - workflow_example1
           - name: galaxy-proxy-prefix
             files: -f docker-compose.yml
-            env: GALAXY_PROXY_PREFIX=/arbitrary_Galaxy-prefix GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost/arbitrary_Galaxy-prefix
+            env: GALAXY_PROXY_PREFIX=/arbitrary_Galaxy-prefix GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost/arbitrary_Galaxy-prefix EXTRA_SKIP_TESTS_BIOBLEND="not test_import_export_workflow_dict and not test_import_export_workflow_from_local_path"
             exclude_test:
-              - workflow_example1
+              - selenium
           - name: galaxy-htcondor
             files: -f docker-compose.yml -f docker-compose.htcondor.yml
-            exclude_test:
-              - workflow_example1
           - name: galaxy-slurm
             files: -f docker-compose.yml -f docker-compose.slurm.yml
             env: SLURM_NODE_COUNT=3
             options: --scale slurm_node=3
-            exclude_test:
-              - workflow_example1
           - name: galaxy-pulsar
             files: -f docker-compose.yml -f docker-compose.pulsar.yml
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job"
             exclude_test:
-              - workflow_example1
-              - workflow_mapping_by_sequencing
-              - workflow_ard
+              - workflow_quality_control
           - name: galaxy-pulsar-mq
             files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job"
             exclude_test:
-              - workflow_example1
-              - workflow_mapping_by_sequencing
-              - workflow_ard
+              - workflow_quality_control
           - name: galaxy-k8s
             files: -f docker-compose.yml -f docker-compose.k8s.yml
-            exclude_test:
-              - bioblend
-              - workflow_ard
-              - workflow_mapping_by_sequencing
-              - workflow_example1
-              - selenium
           - name: galaxy-singularity
             files: -f docker-compose.yml -f docker-compose.singularity.yml
-            exclude_test:
-              - bioblend
-              - workflow_ard
-              - workflow_mapping_by_sequencing
-              - selenium
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver"
           - name: galaxy-pulsar-mq-singularity
             files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml -f docker-compose.singularity.yml
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job and not test_get_container_resolvers and not test_show_container_resolver"
             exclude_test:
-              - bioblend
-              - workflow_ard
-              - workflow_mapping_by_sequencing
-              - selenium
+              - workflow_quality_control
           - name: galaxy-slurm-singularity
             files: -f docker-compose.yml -f docker-compose.slurm.yml -f docker-compose.singularity.yml
-            exclude_test:
-              - bioblend
-              - workflow_ard
-              - workflow_mapping_by_sequencing
-              - selenium
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver"
           - name: galaxy-htcondor-singularity
             files: -f docker-compose.yml -f docker-compose.htcondor.yml -f docker-compose.singularity.yml
-            exclude_test:
-              - bioblend
-              - workflow_ard
-              - workflow_mapping_by_sequencing
-              - selenium
+            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver"
         test:
           - name: bioblend
             files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.bioblend.yml
@@ -85,10 +57,10 @@ jobs:
             workflow: sklearn/ard/ard.ga
             timeout: 60
             second_run: "true"
-          - name: workflow_mapping_by_sequencing
+          - name: workflow_quality_control
             files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
             exit-from: galaxy-workflow-test
-            workflow: training/variant-analysis/mapping-by-sequencing/mapping_by_sequencing.ga
+            workflow: training/sequence-analysis/quality-control/quality_control.ga
             timeout: 60
           - name: workflow_example1
             files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
@@ -164,6 +136,10 @@ jobs:
         shell: bash
         working-directory: ./compose
         continue-on-error: false
+      - name: Fix file names before saving artifacts
+        if: failure()
+        run: |
+          sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \;
       - name: Allow upload-artifact read access
         if: failure()
         run: sudo chmod -R +r ./compose/export/galaxy/database
@@ -180,7 +156,7 @@ jobs:
           sudo rm -rf export/galaxy/database
         working-directory: ./compose
       - name: Run tests a second time
-        if: matrix.test.second_run == 'true' && steps.run_check.run
+        if: matrix.test.second_run == 'true' && steps.run_check.outputs.run
         run: |
           export IMAGE_TAG=ci-testing
           export COMPOSE_DOCKER_CLI_BUILD=1
@@ -206,8 +182,12 @@ jobs:
         shell: bash
         working-directory: ./compose
         continue-on-error: false
+      - name: Fix file names before saving artifacts
+        if: failure() && matrix.test.second_run == 'true'
+        run: |
+          sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \;
       - name: Allow upload-artifact read access
-        if: failure()
+        if: failure() && matrix.test.second_run == 'true'
         run: sudo chmod -R +r ./compose/export/galaxy/database
       - name: Save artifacts for debugging a failed test
         uses: actions/upload-artifact@v4

diff --git a/compose/README.md b/compose/README.md
@@ -395,17 +395,19 @@ The following are settings specific to this docker-compose setup:
 | `SLURM_NODE_HOSTNAME`     | Docker Compose adds a prefix in front of the container names by default. Change this value to the name of your setup and `_slurm_node` (e.g. `compose_slurm_node`) to ensure a correct mapping of the Slurm nodes. |
 
 ### Github Workflow Tests (Branch 24.1)
-| Setup                  | bioblend           | workflow ard       | workflow mapping_by_sequencing | workflow wf3-shed-tools (example1) | selenium           |
-|------------------------|--------------------|--------------------|--------------------------------|------------------------------------|--------------------|
-| Galaxy Base            | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:             | :x:                                | :heavy_check_mark: |
-| Galaxy Proxy Prefix    | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:             | :x:                                | :heavy_check_mark: |
-| HTCondor               | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:             | :x:                                | :heavy_check_mark: |
-| Slurm                  | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:             | :x:                                | :heavy_check_mark: |
-| Pulsar                 | :heavy_check_mark: | :x:                | :x:                            | :x:                                | :heavy_check_mark: |
-| k8s                    | :x:                | :x:                | :x:                            | :x:                                | :x:                |
-| Singularity            | :x:                | :x:                | :x:                            | :heavy_check_mark:                 | :x:                |
-| Slurm + Singularity    | :x:                | :x:                | :x:                            | :heavy_check_mark:                 | :x:                |
-| HTCondor + Singularity | :x:                | :x:                | :x:                            | :heavy_check_mark:                 | :x:                |
+| Setup                  | bioblend           | workflow ard       | workflow quality_control | workflow wf3-shed-tools (example1) | selenium           |
+|------------------------|--------------------|--------------------|--------------------------|------------------------------------|--------------------|
+| Galaxy Base            | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:       | :heavy_check_mark:                 | :heavy_check_mark: |
+| Galaxy Proxy Prefix    | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:       | :heavy_check_mark:                 | :x:                |
+| HTCondor               | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:       | :heavy_check_mark:                 | :heavy_check_mark: |
+| Slurm                  | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:       | :heavy_check_mark:                 | :heavy_check_mark: |
+| Pulsar-REST            | :heavy_check_mark: | :heavy_check_mark: | :x:                      | :heavy_check_mark:                 | :heavy_check_mark: |
+| Pulsar-MQ              | :heavy_check_mark: | :heavy_check_mark: | :x:                      | :heavy_check_mark:                 | :heavy_check_mark: |
+| k8s                    | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:       | :heavy_check_mark:                 | :heavy_check_mark: |
+| Singularity            | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:       | :heavy_check_mark:                 | :heavy_check_mark: |
+| Pulsar-MQ + Singularity| :heavy_check_mark: | :heavy_check_mark: | :x:                      | :heavy_check_mark:                 | :heavy_check_mark: |
+| Slurm + Singularity    | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:       | :heavy_check_mark:                 | :heavy_check_mark: |
+| HTCondor + Singularity | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:       | :heavy_check_mark:                 | :heavy_check_mark: |
 
 
 Implemented: :heavy_check_mark:   

diff --git a/compose/base-images/galaxy-cluster-base/Dockerfile b/compose/base-images/galaxy-cluster-base/Dockerfile
@@ -17,11 +17,11 @@ RUN curl -fsSL https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-current-
     && /usr/bin/common_cleanup.sh
 
 # Install Slurm client
-ENV MUNGER_USER=munge \
+ENV MUNGE_USER=munge \
     MUNGE_UID=1200 \
     MUNGE_GID=1200
-RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \
-    && useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \
+RUN groupadd -r $MUNGE_USER -g $MUNGE_GID \
+    && useradd -u $MUNGE_UID -r -g $MUNGE_USER $MUNGE_USER \
     && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu jammy main" >> /etc/apt/sources.list \
     && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu jammy main" >> /etc/apt/sources.list \
     && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8DE68488997C5C6BA19021136F2CC56412788738 \

diff --git a/compose/base-images/galaxy-container-base/Dockerfile b/compose/base-images/galaxy-container-base/Dockerfile
@@ -27,7 +27,7 @@ FROM ubuntu:22.04 as final
 COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
 
 # Base dependencies
-RUN apt update && apt install --no-install-recommends ca-certificates python3-distutils squashfs-tools -y \
+RUN apt update && apt install --no-install-recommends ca-certificates python3-distutils squashfs-tools tzdata -y \
     && /usr/bin/common_cleanup.sh
 
 # Install Docker

diff --git a/compose/base_config.yml b/compose/base_config.yml
@@ -78,11 +78,20 @@ htcondor_galaxy:
   ALLOW_READ: "*"
   ALLOW_WRITE: "*"
   ALLOW_CLIENT: "*"
+  ALLOW_DAEMON: "*"
   ALLOW_NEGOTIATOR: "*"
   DAEMON_LIST: "MASTER, SCHEDD"
   UID_DOMAIN: "galaxy"
   DISCARD_SESSION_KEYRING_ON_STARTUP: "False"
   TRUST_UID_DOMAIN: "true"
+  SEC_PASSWORD_FILE: "/var/lib/condor/pool_password"
+  SEC_DAEMON_AUTHENTICATION: "REQUIRED"
+  SEC_DAEMON_INTEGRITY: "REQUIRED"
+  SEC_DAEMON_AUTHENTICATION_METHODS: "PASSWORD"
+  SEC_NEGOTIATOR_AUTHENTICATION: "REQUIRED"
+  SEC_NEGOTIATOR_INTEGRITY: "REQUIRED"
+  SEC_NEGOTIATOR_AUTHENTICATION_METHODS: "PASSWORD"
+  SEC_CLIENT_AUTHENTICATION_METHODS: "FS, PASSWORD"
 
 htcondor_master:
   BASE_CGROUP: ""
@@ -101,9 +110,18 @@ htcondor_master:
   ALLOW_READ_COLLECTOR: "*"
   ALLOW_READ_STARTD: "*"
   ALLOW_CLIENT: "*"
+  ALLOW_DAEMON: "*"
   DOCKER_IMAGE_CACHE_SIZE: "20"
   UID_DOMAIN: "galaxy"
   TRUST_UID_DOMAIN: "TRUE"
+  SEC_PASSWORD_FILE: "/var/lib/condor/pool_password"
+  SEC_DAEMON_AUTHENTICATION: "REQUIRED"
+  SEC_DAEMON_INTEGRITY: "REQUIRED"
+  SEC_DAEMON_AUTHENTICATION_METHODS: "PASSWORD"
+  SEC_NEGOTIATOR_AUTHENTICATION: "REQUIRED"
+  SEC_NEGOTIATOR_INTEGRITY: "REQUIRED"
+  SEC_NEGOTIATOR_AUTHENTICATION_METHODS: "PASSWORD"
+  SEC_CLIENT_AUTHENTICATION_METHODS: "FS, PASSWORD"
 
 htcondor_executor:
   CONDOR_HOST: "htcondor-master"
@@ -118,10 +136,19 @@ htcondor_executor:
   ALLOW_READ: "*"
   ALLOW_WRITE: "*"
   ALLOW_CLIENT: "*"
+  ALLOW_DAEMON: "*"
   ALLOW_NEGOTIATOR_SCHEDD: "*"
   ALLOW_WRITE_COLLECTOR: "*"
   ALLOW_WRITE_STARTD: "*"
   ALLOW_READ_COLLECTOR: "*"
   ALLOW_READ_STARTD: "*"
   UID_DOMAIN: "galaxy"
   SCHED_NAME: "htcondor-master"
+  SEC_PASSWORD_FILE: "/var/lib/condor/pool_password"
+  SEC_DAEMON_AUTHENTICATION: "REQUIRED"
+  SEC_DAEMON_INTEGRITY: "REQUIRED"
+  SEC_DAEMON_AUTHENTICATION_METHODS: "PASSWORD"
+  SEC_NEGOTIATOR_AUTHENTICATION: "REQUIRED"
+  SEC_NEGOTIATOR_INTEGRITY: "REQUIRED"
+  SEC_NEGOTIATOR_AUTHENTICATION_METHODS: "PASSWORD"
+  SEC_CLIENT_AUTHENTICATION_METHODS: "FS, PASSWORD"