many things changes, merge of branches for CI/CD is happening soon

ansible.cfg

@@ -4,7 +4,7 @@ inventory = hosts
 retry_files_enabled = false
 vault_password_file = .vault-password.txt
 gather_subset=!hardware
-no_log = True
+no_log = False
 
 # Show diffs of changes
 [diff]

eu_tool_list.yaml

@@ -293,6 +293,7 @@ tools:
   revisions:
   - f46f0e4f75c4
   - d698c222f354
+  - fbf99087e067
   tool_panel_section_label: Text Manipulation
   tool_shed_url: toolshed.g2.bx.psu.edu
 - name: query_tabular
@@ -521,6 +522,7 @@ tools:
   owner: iuc
   revisions:
   - 374cb875d38a
+  - a513e3fbb4c5
   tool_panel_section_label: Join, Subtract and Group
   tool_shed_url: toolshed.g2.bx.psu.edu
 - name: datamash_reverse
@@ -5553,12 +5555,12 @@ tools:
   - fd1f57782683
   tool_panel_section_label: Peak Calling
   tool_shed_url: toolshed.g2.bx.psu.edu
-- name: jamm
-  owner: messersc
-  revisions:
-  - 243f75d0ed6e
-  tool_panel_section_label: Peak Calling
-  tool_shed_url: testtoolshed.g2.bx.psu.edu
+#- name: jamm
+#  owner: messersc
+#  revisions:
+#  - 243f75d0ed6e
+#  tool_panel_section_label: Peak Calling
+#  tool_shed_url: testtoolshed.g2.bx.psu.edu
 - name: piranha
   owner: rnateam
   revisions:

files/galaxy/tools/testing.xml

@@ -1,15 +1,12 @@
 <tool id="testing" name="Big upload test">
     <command>
-        <![CDATA[mv "$input1" "$output1" &&
-	         mv "$input2" "$output2"]]>
+        <![CDATA[mv "$input1" "$output1"]]>
     </command>
     <inputs>
-	    <param name="input1" type="text" label="Input Text1"/>
-            <param name="input2" type="text" label="Input Text2"/>
+        <param name="input1" type="data" format="txt" label="Input Dataset"/>
     </inputs>
     <outputs>
-	    <data name="output1" format="fastq.gz" />
-	    <data name="output2" format="fastq.gz" />
+        <data name="output1" format="txt" />
     </outputs>
 </tool>
 

galaxy.yml

@@ -29,7 +29,7 @@
         mode: 0640
     - name: Install Dependencies
       package:
-        name: ['acl', 'bzip2', 'git', 'make', 'tar', 'python3-bioblend', 'python3-venv', 'python3-setuptools']
+        name: ['acl', 'bzip2', 'git', 'make', 'tar', 'python3-bioblend', 'python3-venv', 'python3-setuptools', 'squashfs-tools']
     - name: Install RHEL/CentOS/Rocky specific dependencies
       package:
         name: ['tmpwatch']
@@ -100,3 +100,28 @@
         path: "{{ galaxy_root }}/server/lib/galaxy/webapps/galaxy/api/job_files.py"
         regexp: '^(\s*?)shutil\.move\(input_file\.name,\s*path\)\s*$'
         replace: '\1os.chmod(shutil.move(input_file.name, path), 0o664)'
+
+    ## Install required Galaxy tools
+    - name: Install required tools - create Ephemeris venv
+      ansible.builtin.shell:
+        cmd: python3 -m venv ~/ephemeris_venv
+      tags: tools
+    - name: Install required tools - install Ephemeris
+      ansible.builtin.shell:
+        cmd: source ~/ephemeris_venv/bin/activate && pip install ephemeris
+      tags: tools
+    - name: Install required tools - install Galaxy tools
+      ansible.builtin.shell:
+        cmd: ~/ephemeris_venv/bin/shed-tools install -g https://{{ inventory_hostname }} -a {{ api_key }} -t regalaxy_tool_list.yaml
+      when: "'galaxy-re' in inventory_hostname"
+      tags: tools
+    - name: Install required tools - install Galaxy tools
+      ansible.builtin.shell:
+        cmd: ~/ephemeris_venv/bin/shed-tools install -g https://{{ inventory_hostname }} -a {{ api_key }} -t umsa_tool_list.yaml
+      when: "'umsa' in inventory_hostname"
+      tags: tools
+    - name: Install required tools - install Galaxy tools
+      ansible.builtin.shell:
+        cmd: ~/ephemeris_venv/bin/shed-tools install -g https://{{ inventory_hostname }} -a {{ api_key }} -t eu_tool_list.yaml
+      when: "'usegalaxy' in inventory_hostname or 'galaxy-cz' in inventory_hostname"
+      tags: tools

group_vars/dbservers.yml

@@ -35,6 +35,6 @@ postgresql_objects_privileges:
     privs: USAGE,SELECT
 
 # PostgreSQL Backups
-postgresql_backup_dir: /data/backups
+postgresql_backup_dir: /rbd/data/backups
 postgresql_backup_local_dir: "{{ '~postgres' | expanduser }}/backups"
 

group_vars/galaxyservers.yml

@@ -30,16 +30,6 @@ galaxy_job_config:
       amqp_publish_retry: true
       amqp_publish_retry_max_retries: 60
       manager: _default_
-    pulsar_gpu_runner:
-      load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner
-      galaxy_url: "https://{{ rabbitmq_hostname }}"
-      amqp_url: "pyamqp://pulsar:{{ rabbitmq_users_password.pulsar }}@{{ rabbitmq_hostname }}:5671/pulsar?ssl=1"
-      amqp_acknowledge: true
-      amqp_ack_republish_time: 1200
-      amqp_consumer_timeout: 2
-      amqp_publish_retry: true
-      amqp_publish_retry_max_retries: 60
-      manager: gpu
     pulsar_tpv_runner:
       load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner
       galaxy_url: "https://{{ rabbitmq_hostname }}"
@@ -86,9 +76,6 @@ galaxy_job_config:
         singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro,$SCRATCHDIR"
         ## Following configuration works!
         # singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro"
-        container_resolvers: 
-          - type: explicit_singularity
-          - type: mulled_singularity
         env:
         # Ensuring a consistent collation environment is good for reproducibility.
         - name: LC_ALL
@@ -99,43 +86,6 @@ galaxy_job_config:
           value: "$SCRATCHDIR"
         - name: TEMP
           value: "$SCRATCHDIR"
-      pulsar_gpu:
-        runner: pulsar_gpu_runner
-        default_file_action: remote_transfer
-        transport: curl
-        dependency_resolution: remote
-        jobs_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-cz/files/staging" 
-        persistence_directory: "/opt/pulsar/files/persistent"
-        remote_metadata: false 
-        rewrite_parameters: true
-        outputs_to_working_directory: false
-        singularity_enabled: true
-        singularity_run_extra_arguments: '--nv'
-        singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,$ALPHAFOLD_DB:/data:ro'
-        env:
-        # The cache directory holds the docker containers that get converted
-        - name: SINGULARITY_CACHEDIR
-          value: "/storage/praha5-elixir/home/galaxyeu/singularity/cache"
-        - name: APPTAINER_CACHEDIR
-          value: "/storage/praha5-elixir/home/galaxyeu/singularity/cache"
-        # Singularity uses a temporary directory to build the squashfs filesystem
-        - name: SINGULARITY_TMPDIR
-          value: "/storage/praha5-elixir/home/galaxyeu/singularity/tmp"
-        - name: APPTAINER_TMPDIR
-          value: "/storage/praha5-elixir/home/galaxyeu/singularity/tmp"
-        # Alphafold specific variables
-        - name: ALPHAFOLD_DB
-          value: "/storage/brno11-elixir/projects/alphafold/alphafold.db-2.3.1"
-        # Default variables
-        - name: TMPDIR
-          value: "$SCRATCHDIR"
-        - name: TMP
-          value: "$SCRATCHDIR"
-        - name: TEMP
-          value: "$SCRATCHDIR"
-        # Ensuring a consistent collation environment is good for reproducibility.
-        - name: LC_ALL
-          value: C
       tpv_dispatcher:
         runner: dynamic
         type: python
@@ -149,6 +99,13 @@ galaxy_job_config:
     groups:
       default: []
       testing: [cores, time]
+  limits:
+    - type: registered_user_concurrent_jobs
+      value: 10
+    - type: anonymous_user_concurrent_jobs
+      value: 1
+    - type: output_size
+      value: 50GB
   tools:
     - class: local # these special tools that aren't parameterized for remote execution - expression tools, upload, etc
       environment: local_env
@@ -178,10 +135,12 @@ galaxy_config:
     cleanup_job: onsuccess
     smtp_server: "rs.cesnet.cz:25"
     error_email_to: "[email protected]"
-    allow_user_creation: False
+    email_from: "[email protected]"
+    allow_user_creation: false
+    require_login: true
     database_connection: "postgresql:///{{ galaxy_db_name }}?host=/var/run/postgresql"
-    file_path: /mnt/data/datasets
-    job_working_directory: /mnt/data/jobs
+    file_path: /rbd/data/datasets
+    job_working_directory: /rbd/data/jobs
     cluster_files_directory: "/rbd/data/pbs"
     short_term_storage_dir: "/rbd/data/short_term_web_storage"
     object_store_store_by: uuid
@@ -221,7 +180,16 @@ galaxy_config:
     tool_data_table_config_path: /cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml
     # Tool Dependencies
     dependency_resolvers_config_file: "{{ galaxy_config_dir }}/dependency_resolvers_conf.xml"
-    containers_resolvers_config_file: "{{ galaxy_config_dir }}/container_resolvers_conf.yml"
+    container_resolvers:
+    -   type: explicit_singularity
+#    -   cache_directory: /cvmfs/singularity.galaxyproject.org/all/
+    -   cache_directory: /rbd/data/singularity
+        type: cached_mulled_singularity
+    -   auto_install: true
+        type: mulled_singularity
+    -   auto_install: false
+        type: build_mulled_singularity   
+        cache_directory: /rbd/data/singularity/mulled
     # Data Library Directories
 #    library_import_dir: "/rbd/data/library_import_dir" # likely to be here
     library_import_dir: /libraries/admin 
@@ -235,7 +203,7 @@ galaxy_config:
     statsd_host: localhost
     statsd_influxdb: true
     # FTP
-    ftp_upload_dir: /mnt/data/uploads
+    ftp_upload_dir: /rbd/data/uploads
     ftp_upload_site: "{{ inventory_hostname }}"
   gravity:
     process_manager: systemd
@@ -297,12 +265,8 @@ galaxy_config_files_public:
 galaxy_config_files:
   - src: files/galaxy/themes.yml
     dest: "{{ galaxy_config.galaxy.themes_config_file }}"
-  - src: files/galaxy/config/tpv_rules_local.yml
-    dest: "{{ tpv_mutable_dir }}/tpv_rules_local.yml"
 
 galaxy_config_templates:
-  - src: templates/galaxy/config/container_resolvers_conf.yml.j2
-    dest: "{{ galaxy_config.galaxy.containers_resolvers_config_file }}"
   - src: templates/galaxy/config/dependency_resolvers_conf.xml
     dest: "{{ galaxy_config.galaxy.dependency_resolvers_config_file }}"
   - src: templates/galaxy/config/job_resource_params_conf.xml.j2
@@ -319,10 +283,11 @@ galaxy_config_templates:
     dest: "{{ galaxy_config.galaxy.tool_config_file[0] }}"
   - src: templates/galaxy/config/local_tool_conf.xml.j2
     dest: "{{ galaxy_config.galaxy.tool_config_file[1] }}"
+  - src: templates/galaxy/config/tpv_rules_local.yml.j2
+    dest: "{{ tpv_mutable_dir }}/tpv_rules_local.yml"
 
 
 galaxy_extra_dirs:
-  - /mnt/data
   - "{{ galaxy_config_dir }}/{{ tpv_config_dir_name }}"
 
 galaxy_extra_privsep_dirs:
@@ -365,7 +330,7 @@ nginx_servers:
   - redirect-ssl
 nginx_ssl_servers:
   - galaxy
-    #  - sentry
+#  - sentry
 nginx_enable_default_server: false
 nginx_conf_http:
   client_max_body_size: 1g
@@ -448,7 +413,7 @@ rabbitmq_users:
 
 # TUS
 galaxy_tusd_port: 1080
-galaxy_tus_upload_store: /rbd/data/tus # /mnt/data/tus
+galaxy_tus_upload_store: /rbd/data/tus
 
 #Redis
 galaxy_additional_venv_packages:

requirements.yml

@@ -19,7 +19,7 @@
   version: 0.0.1
 # CVMFS Support role
 - src: galaxyproject.cvmfs
-  version: 0.2.21
+  version: 0.2.22
 # Singularity/Apptainer
 - src: usegalaxy_eu.apptainer
   version: 0.0.1

roles/galaxyproject.cvmfs/.github/workflows/lint.yml

@@ -7,11 +7,9 @@ jobs:
     name: ansible-lint
     runs-on: ubuntu-latest
     steps:
-      - name: Check out the codebase.
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v2
 
-      - name: Set up Python 3.
-        uses: actions/setup-python@v3
+      - uses: actions/setup-python@v2
         with:
           python-version: '3.x'
 

roles/galaxyproject.cvmfs/.github/workflows/release.yml

@@ -18,10 +18,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the codebase.
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
 
       - name: Set up Python 3.
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v2
         with:
           python-version: '3.x'
 

roles/galaxyproject.cvmfs/.github/workflows/slugger.yml

@@ -24,12 +24,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the codebase.
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
       - name: Set up Python 3.
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v2
         with:
           python-version: '3.x'
 

roles/galaxyproject.cvmfs/meta/.galaxy_install_info

@@ -1,2 +1,2 @@
-install_date: 'Wed 06 Mar 2024 01:49:34 PM '
-version: 0.2.21
+install_date: 'Sun 26 May 2024 09:19:28 PM '
+version: 0.2.22

roles/galaxyproject.cvmfs/tasks/localproxy.yml

@@ -17,7 +17,7 @@
 #       {{ inventory_hostname }} 127.0.0.1
 #     dest: /etc/squid/hosts
 #   notify:
-#     - restart squid
+#     - Restart squid
 
 - name: Include firewall tasks
   ansible.builtin.include_tasks: firewall.yml

roles/galaxyproject.cvmfs/tasks/squid.yml

@@ -6,7 +6,7 @@
     backup: true
     mode: 0644
   notify:
-    - restart squid
+    - Restart squid
 
 - name: Fix cache directory permission
   ansible.builtin.file:
@@ -23,7 +23,7 @@
   become_user: "{{ cvmfs_squid_user }}"
   ansible.builtin.command: squid -z
   args:
-    creates: "{{ cvmfs_stratum1_cache_dir }}/00"
+    creates: "{{ cvmfs_stratum1_cache_dir.dir }}/00"
   when: cvmfs_stratum1_cache_dir is defined
 
 - name: Ensure squid is enabled and started

roles/galaxyproject.cvmfs/tasks/stratum0.yml

@@ -46,7 +46,7 @@
     creates: /srv/cvmfs/{{ item.repository }}
   with_items: "{{ cvmfs_repositories }}"
   notify:
-    - restart apache
+    - Restart apache
 
 - name: Ensure repositories are imported
   ansible.builtin.command: |
@@ -55,7 +55,7 @@
     creates: /etc/cvmfs/repositories.d/{{ item.repository }}
   with_items: "{{ cvmfs_repositories }}"
   notify:
-    - restart apache
+    - Restart apache
 
 - name: Include repository server options tasks
   ansible.builtin.include_tasks: options.yml

roles/galaxyproject.cvmfs/tasks/stratum1.yml

@@ -16,7 +16,7 @@
     backup: true
   when: cvmfs_config_apache
   notify:
-    - reload apache
+    - Reload apache
 
 - name: Include stratumN tasks
   ansible.builtin.include_tasks: stratumN.yml
@@ -53,7 +53,7 @@
   loop: "{{ cvmfs_repositories }}"
   register: __cvmfs_add_replica_result
   notify:
-    - restart apache
+    - Restart apache
 
 # Ideally we could use item.stratum0_url_scheme directly in `cvmfs_server add-replica` above, but it appears not to
 # support it, so we instead have to change it after the fact