Galaxy release_24.1 (#58)

* change of galaxy_commit_id value for initial test in gce

* Update README.md

* Update test_playbook.yaml

* Update requirements.yml

* simplifiy  ansible_galaxy_tools role (24.0_addon)

* inactivate BIOBLEND_GALAXY_API_KEY for test

* Update test_playbook.yaml

* clean code after fixing the ansible.tools role

* Update mississippi_dev job_conf.yml

This update is just done using the actual running job_conf.yml of mississippi_dev and copying it in the corresponding file of Galaxy expand (pointed by the commit)

* change Galaxy commit for all environments to 24.0

Now that dev_gce passes all tests in GitHub we apply version change to all environments, using the 000_cross_env_vars files

* Create mississippi_dev_tool_list.yml

.venv must contains ephemeris (pip install ephemeris)

* Update conect job_conf.yml

drop local environment/local_runner which probably causes crashes of workflows

* Update conect_tool_list.yml

from get-tool-list -g usegalaxy.sorbonne-universite.fr ... and cat export file

* Update conect Welcome html page variables

* new settings in test in dev_mississippi and in conect

* maj with CNVkit

* maj with cnvkit tool suite

* interactivetools_enable and enable_tool_source_display are now common to playbook

* Update mississippi_tool_list.yml

* remove local environment from job_conf.yml

As tested in conect environment

* Update Mississippi html alerts

* target lumpy_smoove to cluster_4 destination in Mississippi[2]

  - id: "lumpy_smoove"
    destination: "cluster_4"

* assign a destination cluster8 to picard_AddOrReplaceReadGroups

picard_AddOrReplaceReadGroups

* update messages for artbio environment of the playbook

.github/workflows/test_playbook.yaml

@@ -7,8 +7,7 @@ on:
       - master
 env:
   GALAXY_USER: [email protected]
-  GALAXY_USER_PASSWD: artbio2025
-  BIOBLEND_GALAXY_API_KEY: "myverysecretapikey"
+  GALAXY_USER_PASSWD: artbio2024
   BIOBLEND_GALAXY_URL: "http://127.0.0.1:80"
   BIOBLEND_TEST_JOB_TIMEOUT: "240"
 
@@ -42,7 +41,7 @@ jobs:
     - name: Install ansible 3.0 and bioblend in ubuntu 20.04
       run: |
         python3 -m pip install ansible==3.0
-        python3 -m pip install --ignore-installed https://github.com/galaxyproject/bioblend/archive/refs/tags/v1.2.0.zip pytest
+        python3 -m pip install --ignore-installed https://github.com/galaxyproject/bioblend/archive/refs/tags/v1.3.0.zip pytest
     
     - name: Display pip python and ansible settings
       run: |
@@ -73,14 +72,12 @@ jobs:
                          --skip-tags galaxy_build_client \
                          playbook.yml
 
-    - name: Sleep for 30 secs and check galaxyctl status
+    - name: Sleep for 60 secs and check galaxyctl status
       run: |
-        sudo sed -E -i 's/www-data/galaxy/' /etc/nginx/nginx.conf
         sudo systemctl restart nginx.service
+        sudo galaxyctl status
         sudo galaxyctl graceful
-        echo "waiting 30 sec" && sleep 30
-        sudo cat /etc/nginx/nginx.conf
-        sudo cat /home/galaxy/galaxy/config/galaxy.yml
+        echo "waiting 60 sec" && sleep 60
         sudo galaxyctl status
 
     - name: check Galaxy API is alive
@@ -90,6 +87,11 @@ jobs:
     - name: ansible installs galaxy tools
       run: ansible-playbook --extra-vars RUNNER_ALLOW_RUNASROOT="1" -e ansible_user="runner" install_tools.yml
 
+    - name: retrieve api key and reference it in glob env variable
+      run: |
+        BIOBLEND_GALAXY_API_KEY=`cat ./apikey.txt`
+        echo "BIOBLEND_GALAXY_API_KEY=$BIOBLEND_GALAXY_API_KEY" >> $GITHUB_ENV
+
     - name: Bioblend tests
       run: |
         cd /home/runner/.local/lib/python3.8/site-packages/bioblend/_tests

README.md

@@ -10,13 +10,18 @@ Documentation is coming soon.
 Requirements
 ```
 
-Python >= 3.7 
-ansible >= 2.10.1
+Python >= 3.8 
+ansible >= 3.0
 ```
 
 ### Example of use
 
 ```
+apt update
+apt install python3-pip python3-virtualenv
+pip install ansible==3.0
+git clone https://github.com/ARTbio/galaxyXpand.git
+cd galaxyXpand
 ansible-galaxy install -r requirements.yml -p roles/
 ansible-playbook -i environments/dev_gce/hosts playbook.yml
 ```
@@ -27,7 +32,7 @@ Then,
 ```
 ansible-playbook -i environments/dev_gce/hosts install_tools.yml
 ```
-installs 3 sample tools described in [tool_list.yaml.sample](https://github.com/ARTbio/ansible-galaxy-tools/blob/galaxyXpand/files/tool_list.yaml.sample)
+installs 2 tools described as described in [tool_list.yaml.sample](https://github.com/ARTbio/ansible-galaxy-tools/blob/galaxyXpand/files/tool_list.yaml.sample)
 
 ----
 ```

environments/000_cross_env_vars

@@ -15,7 +15,7 @@ nginx_conf_user: "www-data galaxy"
 galaxy_manage_systemd: true
 
 # Galaxy common to all hosts
-galaxy_commit_id: release_23.2
+galaxy_commit_id: release_24.0
 galaxy_user_name: galaxy
 galaxy_db_name: galaxy
 galaxy_config_perms: 0664
@@ -65,6 +65,8 @@ common_galaxy_config:
     expose_user_name: true
     expose_dataset_path: true
     expose_potentially_sensitive_job_metrics: true
+    enable_tool_source_display: true
+    interactivetools_enable: true
     # Debugging
     cleanup_job: onsuccess
     allow_user_impersonation: true

environments/ARTbio/group_vars/all/galaxy

@@ -70,7 +70,7 @@ galaxy_config:
 # Welcome html page variables
 _artbio_info: |
   <h4>
-  <strong> ARTbio server recently upgraded to release 23.2
+  <strong> ARTbio server recently upgraded to release 24.0 (2024-06-29)
   <br>
   Please report any unusual bugs</strong>
   </h4>

environments/Conect/files/galaxy/config/conect_tool_list.yml

@@ -229,12 +229,12 @@ tools:
   tool_panel_section_id: samtools
   tool_panel_section_label: SAMtools
   tool_shed_url: toolshed.g2.bx.psu.edu
-- name: crossmap_bed
+- name: bedtools
   owner: iuc
   tool_panel_section_id: bedtools
   tool_panel_section_label: BEDtools
   tool_shed_url: toolshed.g2.bx.psu.edu
-- name: bedtools
+- name: crossmap_bed
   owner: iuc
   tool_panel_section_id: bedtools
   tool_panel_section_label: BEDtools
@@ -644,6 +644,11 @@ tools:
   tool_panel_section_id: machine_learning
   tool_panel_section_label: Machine Learning
   tool_shed_url: toolshed.g2.bx.psu.edu
+- name: ez_histograms
+  owner: artbio
+  tool_panel_section_id: stats
+  tool_panel_section_label: Statistics
+  tool_shed_url: toolshed.g2.bx.psu.edu
 - name: table_compute
   owner: iuc
   tool_panel_section_id: stats
@@ -1379,6 +1384,91 @@ tools:
   tool_panel_section_id: cnv
   tool_panel_section_label: CNV
   tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_target
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_antitarget
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_breaks
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_access
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_scatter
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_autobin
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_segment
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_batch
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_heatmap
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_sex
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_fix
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_coverage
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_call
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_segmetrics
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_reference
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_genemetrics
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
+- name: cnvkit_diagram
+  owner: iuc
+  tool_panel_section_id: cnvkit
+  tool_panel_section_label: CNVkit
+  tool_shed_url: toolshed.g2.bx.psu.edu
 - name: data_manager_bowtie_index_builder
   owner: iuc
   tool_panel_section_id: None

environments/Conect/files/galaxy/config/job_conf.yml

@@ -2,9 +2,6 @@
 # Do not edit this file manually. Any changes will be automatically reverted.
 
 runners:
-  local_runner:
-    load: galaxy.jobs.runners.local:LocalJobRunner
-    workers: 8
   slurm:
     load: galaxy.jobs.runners.slurm:SlurmJobRunner
     drmaa_library_path: /usr/lib/slurm-drmaa/lib/libdrmaa.so
@@ -21,13 +18,6 @@ handling:
 execution:
   default: cluster_1
   environments:
-    local_env:
-      runner: local_runner
-      tmp_dir: true
-    gce_multicore:
-      runner: local_runner
-      local_slots: 2
-      embed_metadata_in_job: true
     cluster_1:
       runner: slurm
       nativeSpecification: "--partition=debug --ntasks=1"
@@ -76,17 +66,17 @@ limits:
   - type: anonymous_user_concurrent_jobs
     value: 1
   - type: registered_user_concurrent_jobs
-    value: 70
+    value: 40
 
   - type: environment_total_concurrent_jobs
     id: cluster_1
-    value: 80
+    value: 40
   - type: environment_total_concurrent_jobs
     id: cluster_2
-    value: 50
+    value: 30
   - type: environment_total_concurrent_jobs
     id: cluster_4
-    value: 18
+    value: 22
   - type: environment_total_concurrent_jobs
     id: cluster_8
     value: 10
@@ -120,7 +110,7 @@ limits:
 
 tools:
   - class: local # these special tools that aren't parameterized for remote execution - expression tools, upload, etc
-    environment: local_env
+    environment: "cluster_1"
 
   - id: "arriba"
     destination: "cluster_8"
@@ -152,6 +142,9 @@ tools:
   - id: "bwa"
     destination: "cluster_16"
 
+  - id: "cnvkit_batch"
+    destination: "cluster_4"
+
   - id: "deeptools_bam_coverage"
     destination: "cluster_4"
 

environments/Conect/group_vars/all/galaxy

@@ -54,6 +54,8 @@ galaxy_config:
           6636366361643132663339333535343462653261636436663531
     # Automation / Ease of Use / User-facing features
     require_login: true
+    user_activation_on: true
+    allow_user_creation: false
     # Miscellaneous
     ftp_upload_dir: /home/galaxy/galaxy/database/ftp
     ftp_upload_site: ftp://134.157.183.195
@@ -71,9 +73,9 @@ galaxy_config:
 
 # Welcome html page variables
 _conect_info: |
-  <strong>This is a text message
+  <strong>Galaxy software was upgraded to
   <br>
-  when needing user information</strong>
+  release_24.0 (2024-06-17)</strong>
 conect_info: "{{ _conect_info | indent(width=10, first=True) }}"
 
 # deploy galaxy configuration files

environments/Mississippi/files/galaxy/config/job_conf.yml

@@ -2,9 +2,6 @@
 # Do not edit this file manually. Any changes will be automatically reverted.
 
 runners:
-  local_runner:
-    load: galaxy.jobs.runners.local:LocalJobRunner
-    workers: 8
   slurm:
     load: galaxy.jobs.runners.slurm:SlurmJobRunner
     drmaa_library_path: /usr/lib/slurm-drmaa/lib/libdrmaa.so
@@ -21,13 +18,6 @@ handling:
 execution:
   default: cluster_1
   environments:
-    local_env:
-      runner: local_runner
-      tmp_dir: true
-    gce_multicore:
-      runner: local_runner
-      local_slots: 2
-      embed_metadata_in_job: true
     cluster_1:
       runner: slurm
       nativeSpecification: "--partition=debug --ntasks=1"
@@ -75,7 +65,7 @@ limits:
 
 tools:
   - class: local # these special tools that aren't parameterized for remote execution - expression tools, upload, etc
-    environment: local_env
+    environment: "cluster_1"
 
   - id: "abyss-pe"
     destination: "cluster_8"
@@ -146,6 +136,9 @@ tools:
   - id: "lofreq_call"
     destination: "cluster_8"
 
+  - id: "lumpy_smoove"
+    destination: "cluster_4"
+
   - id: "macs2_callpeak"
     destination: "cluster_8"
 
@@ -182,6 +175,9 @@ tools:
   - id: "picard_MergeSamFiles"
     destination: "cluster_8"
 
+  - id: "picard_AddOrReplaceReadGroups"
+    destination: "cluster_8"
+
   - id: "picard_SortSam"
     destination: "cluster_8"