From 009c95288f449c612b14ce0f1981ab7f3396eb29 Mon Sep 17 00:00:00 2001 From: Christophe Antoniewski Date: Wed, 3 Jul 2024 11:26:22 +0200 Subject: [PATCH] Galaxy release_24.1 (#58) * change of galaxy_commit_id value for initial test in gce * Update README.md * Update test_playbook.yaml * Update requirements.yml * simplifiy ansible_galaxy_tools role (24.0_addon) * inactivate BIOBLEND_GALAXY_API_KEY for test * Update test_playbook.yaml * clean code after fixing the ansible.tools role * Update mississippi_dev job_conf.yml This update is just done using the actual running job_conf.yml of mississippi_dev and copying it in the corresponding file of Galaxy expand (pointed by the commit) * change Galaxy commit for all environments to 24.0 Now that dev_gce passes all tests in GitHub we apply version change to all environments, using the 000_cross_env_vars files * Create mississippi_dev_tool_list.yml .venv must contains ephemeris (pip install ephemeris) * Update conect job_conf.yml drop local environment/local_runner which probably causes crashes of workflows * Update conect_tool_list.yml from get-tool-list -g usegalaxy.sorbonne-universite.fr ... and cat export file * Update conect Welcome html page variables * new settings in test in dev_mississippi and in conect * maj with CNVkit * maj with cnvkit tool suite * interactivetools_enable and enable_tool_source_display are now common to playbook * Update mississippi_tool_list.yml * remove local environment from job_conf.yml As tested in conect environment * Update Mississippi html alerts * target lumpy_smoove to cluster_4 destination in Mississippi[2] - id: "lumpy_smoove" destination: "cluster_4" * assign a destination cluster8 to picard_AddOrReplaceReadGroups picard_AddOrReplaceReadGroups * update messages for artbio environment of the playbook --- .github/workflows/test_playbook.yaml | 18 +- README.md | 11 +- environments/000_cross_env_vars | 4 +- environments/ARTbio/group_vars/all/galaxy | 2 +- .../files/galaxy/config/conect_tool_list.yml | 94 +++++- .../Conect/files/galaxy/config/job_conf.yml | 23 +- environments/Conect/group_vars/all/galaxy | 6 +- .../files/galaxy/config/job_conf.yml | 18 +- .../galaxy/config/mississippi_tool_list.yml | 45 ++- .../Mississippi/group_vars/all/galaxy | 4 +- environments/dev_gce/group_vars/all/galaxy | 2 +- .../files/galaxy/config/job_conf.yml | 16 +- .../config/mississippi_dev_tool_list.yml | 185 ++++++++++++ .../dev_mississippi/group_vars/all/galaxy | 5 + requirements.yml | 2 +- scripts/manage_bootstrap_user.py | 279 ------------------ 16 files changed, 381 insertions(+), 333 deletions(-) create mode 100644 environments/dev_mississippi/files/galaxy/config/mississippi_dev_tool_list.yml delete mode 100644 scripts/manage_bootstrap_user.py diff --git a/.github/workflows/test_playbook.yaml b/.github/workflows/test_playbook.yaml index 0ef63ce..2b33ea3 100644 --- a/.github/workflows/test_playbook.yaml +++ b/.github/workflows/test_playbook.yaml @@ -7,8 +7,7 @@ on: - master env: GALAXY_USER: tooladmin@galaxy.org - GALAXY_USER_PASSWD: artbio2025 - BIOBLEND_GALAXY_API_KEY: "myverysecretapikey" + GALAXY_USER_PASSWD: artbio2024 BIOBLEND_GALAXY_URL: "http://127.0.0.1:80" BIOBLEND_TEST_JOB_TIMEOUT: "240" @@ -42,7 +41,7 @@ jobs: - name: Install ansible 3.0 and bioblend in ubuntu 20.04 run: | python3 -m pip install ansible==3.0 - python3 -m pip install --ignore-installed https://github.com/galaxyproject/bioblend/archive/refs/tags/v1.2.0.zip pytest + python3 -m pip install --ignore-installed https://github.com/galaxyproject/bioblend/archive/refs/tags/v1.3.0.zip pytest - name: Display pip python and ansible settings run: | @@ -73,14 +72,12 @@ jobs: --skip-tags galaxy_build_client \ playbook.yml - - name: Sleep for 30 secs and check galaxyctl status + - name: Sleep for 60 secs and check galaxyctl status run: | - sudo sed -E -i 's/www-data/galaxy/' /etc/nginx/nginx.conf sudo systemctl restart nginx.service + sudo galaxyctl status sudo galaxyctl graceful - echo "waiting 30 sec" && sleep 30 - sudo cat /etc/nginx/nginx.conf - sudo cat /home/galaxy/galaxy/config/galaxy.yml + echo "waiting 60 sec" && sleep 60 sudo galaxyctl status - name: check Galaxy API is alive @@ -90,6 +87,11 @@ jobs: - name: ansible installs galaxy tools run: ansible-playbook --extra-vars RUNNER_ALLOW_RUNASROOT="1" -e ansible_user="runner" install_tools.yml + - name: retrieve api key and reference it in glob env variable + run: | + BIOBLEND_GALAXY_API_KEY=`cat ./apikey.txt` + echo "BIOBLEND_GALAXY_API_KEY=$BIOBLEND_GALAXY_API_KEY" >> $GITHUB_ENV + - name: Bioblend tests run: | cd /home/runner/.local/lib/python3.8/site-packages/bioblend/_tests diff --git a/README.md b/README.md index f5daf7c..1e282d0 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,18 @@ Documentation is coming soon. Requirements ``` -Python >= 3.7 -ansible >= 2.10.1 +Python >= 3.8 +ansible >= 3.0 ``` ### Example of use ``` +apt update +apt install python3-pip python3-virtualenv +pip install ansible==3.0 +git clone https://github.com/ARTbio/galaxyXpand.git +cd galaxyXpand ansible-galaxy install -r requirements.yml -p roles/ ansible-playbook -i environments/dev_gce/hosts playbook.yml ``` @@ -27,7 +32,7 @@ Then, ``` ansible-playbook -i environments/dev_gce/hosts install_tools.yml ``` -installs 3 sample tools described in [tool_list.yaml.sample](https://github.com/ARTbio/ansible-galaxy-tools/blob/galaxyXpand/files/tool_list.yaml.sample) +installs 2 tools described as described in [tool_list.yaml.sample](https://github.com/ARTbio/ansible-galaxy-tools/blob/galaxyXpand/files/tool_list.yaml.sample) ---- ``` diff --git a/environments/000_cross_env_vars b/environments/000_cross_env_vars index f7a8766..f92307b 100644 --- a/environments/000_cross_env_vars +++ b/environments/000_cross_env_vars @@ -15,7 +15,7 @@ nginx_conf_user: "www-data galaxy" galaxy_manage_systemd: true # Galaxy common to all hosts -galaxy_commit_id: release_23.2 +galaxy_commit_id: release_24.0 galaxy_user_name: galaxy galaxy_db_name: galaxy galaxy_config_perms: 0664 @@ -65,6 +65,8 @@ common_galaxy_config: expose_user_name: true expose_dataset_path: true expose_potentially_sensitive_job_metrics: true + enable_tool_source_display: true + interactivetools_enable: true # Debugging cleanup_job: onsuccess allow_user_impersonation: true diff --git a/environments/ARTbio/group_vars/all/galaxy b/environments/ARTbio/group_vars/all/galaxy index 4a14b68..ecbb8cd 100644 --- a/environments/ARTbio/group_vars/all/galaxy +++ b/environments/ARTbio/group_vars/all/galaxy @@ -70,7 +70,7 @@ galaxy_config: # Welcome html page variables _artbio_info: |

- ARTbio server recently upgraded to release 23.2 + ARTbio server recently upgraded to release 24.0 (2024-06-29)
Please report any unusual bugs

diff --git a/environments/Conect/files/galaxy/config/conect_tool_list.yml b/environments/Conect/files/galaxy/config/conect_tool_list.yml index 6ee9a62..d6a37c9 100644 --- a/environments/Conect/files/galaxy/config/conect_tool_list.yml +++ b/environments/Conect/files/galaxy/config/conect_tool_list.yml @@ -229,12 +229,12 @@ tools: tool_panel_section_id: samtools tool_panel_section_label: SAMtools tool_shed_url: toolshed.g2.bx.psu.edu -- name: crossmap_bed +- name: bedtools owner: iuc tool_panel_section_id: bedtools tool_panel_section_label: BEDtools tool_shed_url: toolshed.g2.bx.psu.edu -- name: bedtools +- name: crossmap_bed owner: iuc tool_panel_section_id: bedtools tool_panel_section_label: BEDtools @@ -644,6 +644,11 @@ tools: tool_panel_section_id: machine_learning tool_panel_section_label: Machine Learning tool_shed_url: toolshed.g2.bx.psu.edu +- name: ez_histograms + owner: artbio + tool_panel_section_id: stats + tool_panel_section_label: Statistics + tool_shed_url: toolshed.g2.bx.psu.edu - name: table_compute owner: iuc tool_panel_section_id: stats @@ -1379,6 +1384,91 @@ tools: tool_panel_section_id: cnv tool_panel_section_label: CNV tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_target + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_antitarget + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_breaks + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_access + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_scatter + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_autobin + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_segment + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_batch + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_heatmap + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_sex + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_fix + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_coverage + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_call + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_segmetrics + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_reference + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_genemetrics + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cnvkit_diagram + owner: iuc + tool_panel_section_id: cnvkit + tool_panel_section_label: CNVkit + tool_shed_url: toolshed.g2.bx.psu.edu - name: data_manager_bowtie_index_builder owner: iuc tool_panel_section_id: None diff --git a/environments/Conect/files/galaxy/config/job_conf.yml b/environments/Conect/files/galaxy/config/job_conf.yml index bc314ad..9b1ae53 100644 --- a/environments/Conect/files/galaxy/config/job_conf.yml +++ b/environments/Conect/files/galaxy/config/job_conf.yml @@ -2,9 +2,6 @@ # Do not edit this file manually. Any changes will be automatically reverted. runners: - local_runner: - load: galaxy.jobs.runners.local:LocalJobRunner - workers: 8 slurm: load: galaxy.jobs.runners.slurm:SlurmJobRunner drmaa_library_path: /usr/lib/slurm-drmaa/lib/libdrmaa.so @@ -21,13 +18,6 @@ handling: execution: default: cluster_1 environments: - local_env: - runner: local_runner - tmp_dir: true - gce_multicore: - runner: local_runner - local_slots: 2 - embed_metadata_in_job: true cluster_1: runner: slurm nativeSpecification: "--partition=debug --ntasks=1" @@ -76,17 +66,17 @@ limits: - type: anonymous_user_concurrent_jobs value: 1 - type: registered_user_concurrent_jobs - value: 70 + value: 40 - type: environment_total_concurrent_jobs id: cluster_1 - value: 80 + value: 40 - type: environment_total_concurrent_jobs id: cluster_2 - value: 50 + value: 30 - type: environment_total_concurrent_jobs id: cluster_4 - value: 18 + value: 22 - type: environment_total_concurrent_jobs id: cluster_8 value: 10 @@ -120,7 +110,7 @@ limits: tools: - class: local # these special tools that aren't parameterized for remote execution - expression tools, upload, etc - environment: local_env + environment: "cluster_1" - id: "arriba" destination: "cluster_8" @@ -152,6 +142,9 @@ tools: - id: "bwa" destination: "cluster_16" + - id: "cnvkit_batch" + destination: "cluster_4" + - id: "deeptools_bam_coverage" destination: "cluster_4" diff --git a/environments/Conect/group_vars/all/galaxy b/environments/Conect/group_vars/all/galaxy index adb74d4..6b22976 100644 --- a/environments/Conect/group_vars/all/galaxy +++ b/environments/Conect/group_vars/all/galaxy @@ -54,6 +54,8 @@ galaxy_config: 6636366361643132663339333535343462653261636436663531 # Automation / Ease of Use / User-facing features require_login: true + user_activation_on: true + allow_user_creation: false # Miscellaneous ftp_upload_dir: /home/galaxy/galaxy/database/ftp ftp_upload_site: ftp://134.157.183.195 @@ -71,9 +73,9 @@ galaxy_config: # Welcome html page variables _conect_info: | - This is a text message + Galaxy software was upgraded to
- when needing user information + release_24.0 (2024-06-17) conect_info: "{{ _conect_info | indent(width=10, first=True) }}" # deploy galaxy configuration files diff --git a/environments/Mississippi/files/galaxy/config/job_conf.yml b/environments/Mississippi/files/galaxy/config/job_conf.yml index ce66748..23116eb 100644 --- a/environments/Mississippi/files/galaxy/config/job_conf.yml +++ b/environments/Mississippi/files/galaxy/config/job_conf.yml @@ -2,9 +2,6 @@ # Do not edit this file manually. Any changes will be automatically reverted. runners: - local_runner: - load: galaxy.jobs.runners.local:LocalJobRunner - workers: 8 slurm: load: galaxy.jobs.runners.slurm:SlurmJobRunner drmaa_library_path: /usr/lib/slurm-drmaa/lib/libdrmaa.so @@ -21,13 +18,6 @@ handling: execution: default: cluster_1 environments: - local_env: - runner: local_runner - tmp_dir: true - gce_multicore: - runner: local_runner - local_slots: 2 - embed_metadata_in_job: true cluster_1: runner: slurm nativeSpecification: "--partition=debug --ntasks=1" @@ -75,7 +65,7 @@ limits: tools: - class: local # these special tools that aren't parameterized for remote execution - expression tools, upload, etc - environment: local_env + environment: "cluster_1" - id: "abyss-pe" destination: "cluster_8" @@ -146,6 +136,9 @@ tools: - id: "lofreq_call" destination: "cluster_8" + - id: "lumpy_smoove" + destination: "cluster_4" + - id: "macs2_callpeak" destination: "cluster_8" @@ -182,6 +175,9 @@ tools: - id: "picard_MergeSamFiles" destination: "cluster_8" + - id: "picard_AddOrReplaceReadGroups" + destination: "cluster_8" + - id: "picard_SortSam" destination: "cluster_8" diff --git a/environments/Mississippi/files/galaxy/config/mississippi_tool_list.yml b/environments/Mississippi/files/galaxy/config/mississippi_tool_list.yml index 95b0418..0393757 100644 --- a/environments/Mississippi/files/galaxy/config/mississippi_tool_list.yml +++ b/environments/Mississippi/files/galaxy/config/mississippi_tool_list.yml @@ -299,6 +299,11 @@ tools: tool_panel_section_id: stats tool_panel_section_label: Statistics tool_shed_url: toolshed.g2.bx.psu.edu +- name: ggplot2_point + owner: iuc + tool_panel_section_id: plots + tool_panel_section_label: Graph/Display Data + tool_shed_url: toolshed.g2.bx.psu.edu - name: high_dim_heatmap owner: artbio tool_panel_section_id: plots @@ -709,6 +714,11 @@ tools: tool_panel_section_id: nuclear_3d_organisation tool_panel_section_label: Nuclear 3D organisation tool_shed_url: toolshed.g2.bx.psu.edu +- name: trinity + owner: iuc + tool_panel_section_id: ngs:_assembly + tool_panel_section_label: 'NGS: Assembly' + tool_shed_url: toolshed.g2.bx.psu.edu - name: trinity_samples_qccheck owner: iuc tool_panel_section_id: ngs:_assembly @@ -754,11 +764,6 @@ tools: tool_panel_section_id: ngs:_assembly tool_panel_section_label: 'NGS: Assembly' tool_shed_url: toolshed.g2.bx.psu.edu -- name: trinity - owner: iuc - tool_panel_section_id: ngs:_assembly - tool_panel_section_label: 'NGS: Assembly' - tool_shed_url: toolshed.g2.bx.psu.edu - name: spades owner: nml tool_panel_section_id: ngs:_assembly @@ -1209,6 +1214,11 @@ tools: tool_panel_section_id: ngs:_rna_analysis tool_panel_section_label: 'NGS: RNA Analysis' tool_shed_url: toolshed.g2.bx.psu.edu +- name: repenrich2 + owner: artbio + tool_panel_section_id: transposon_detection + tool_panel_section_label: Transposon Detection + tool_shed_url: toolshed.g2.bx.psu.edu - name: pindel owner: artbio tool_panel_section_id: transposon_detection @@ -1554,6 +1564,31 @@ tools: tool_panel_section_id: single-cell_rnaseq tool_panel_section_label: Single-cell RNAseq tool_shed_url: toolshed.g2.bx.psu.edu +- name: umi_tools_whitelist + owner: iuc + tool_panel_section_id: umi_tools + tool_panel_section_label: UMI Tools + tool_shed_url: toolshed.g2.bx.psu.edu +- name: umi_tools_group + owner: iuc + tool_panel_section_id: umi_tools + tool_panel_section_label: UMI Tools + tool_shed_url: toolshed.g2.bx.psu.edu +- name: umi_tools_count + owner: iuc + tool_panel_section_id: umi_tools + tool_panel_section_label: UMI Tools + tool_shed_url: toolshed.g2.bx.psu.edu +- name: umi_tools_extract + owner: iuc + tool_panel_section_id: umi_tools + tool_panel_section_label: UMI Tools + tool_shed_url: toolshed.g2.bx.psu.edu +- name: umi_tools_dedup + owner: iuc + tool_panel_section_id: umi_tools + tool_panel_section_label: UMI Tools + tool_shed_url: toolshed.g2.bx.psu.edu - name: ceas owner: pjbriggs tool_panel_section_id: None diff --git a/environments/Mississippi/group_vars/all/galaxy b/environments/Mississippi/group_vars/all/galaxy index 0bfc73f..a7fcbe3 100644 --- a/environments/Mississippi/group_vars/all/galaxy +++ b/environments/Mississippi/group_vars/all/galaxy @@ -70,9 +70,9 @@ galaxy_config: # Welcome html page variables _Mississippi_info: | - Mississippi recently upgraded to release 23.2 + On 2024-06-23, Mississippi was upgraded to release 24.0
- Please report any unusual bugs + Please report any unusual bugs with tools or workflows Mississippi_info: "{{ _Mississippi_info | indent(width=12, first=True) }}" # deploy galaxy configuration files diff --git a/environments/dev_gce/group_vars/all/galaxy b/environments/dev_gce/group_vars/all/galaxy index bf05e75..d739cf9 100644 --- a/environments/dev_gce/group_vars/all/galaxy +++ b/environments/dev_gce/group_vars/all/galaxy @@ -2,7 +2,7 @@ # Galaxy # development branch -galaxy_commit_id: release_23.1 +galaxy_commit_id: release_24.0 galaxy_config: galaxy: diff --git a/environments/dev_mississippi/files/galaxy/config/job_conf.yml b/environments/dev_mississippi/files/galaxy/config/job_conf.yml index 4e5ab64..23fcd85 100644 --- a/environments/dev_mississippi/files/galaxy/config/job_conf.yml +++ b/environments/dev_mississippi/files/galaxy/config/job_conf.yml @@ -57,6 +57,9 @@ tools: - id: "bowtie_index_builder_data_manager" destination: "cluster_1" + - id: "bwa_mem" + destination: "cluster_8" + - id: "retrieve_fasta_from_NCBI" destination: "cluster_1" @@ -69,6 +72,9 @@ tools: - id: "bowtie_wrapper" destination: "cluster_8" + - id: "bowtie2" + destination: "cluster_8" + - id: "fastqc" destination: "cluster_2" @@ -76,7 +82,7 @@ tools: destination: "cluster_2" - id: "picard_SamToFastq" - destination: "cluster_1" + destination: "cluster_1" - id: "repenrich" destination: "cluster_8" @@ -87,6 +93,9 @@ tools: - id: "rna_starsolo" destination: "cluster_8" + - id: "sambamba_sample_or_filter" + destination: "cluster_8" + - id: "samtools_cram_to_bam" destination: "cluster_4" @@ -105,6 +114,9 @@ tools: - id: "samtool_filter2" destination: "cluster_2" + - id: "seqtk_sample" + destination: "cluster_8" + - id: "small_rna_maps" destination: "cluster_2" @@ -112,4 +124,4 @@ tools: destination: "cluster_4" - id: "tp_grep_tool" - destination: "cluster_1" + destination: "cluster_1" diff --git a/environments/dev_mississippi/files/galaxy/config/mississippi_dev_tool_list.yml b/environments/dev_mississippi/files/galaxy/config/mississippi_dev_tool_list.yml new file mode 100644 index 0000000..eea2f3c --- /dev/null +++ b/environments/dev_mississippi/files/galaxy/config/mississippi_dev_tool_list.yml @@ -0,0 +1,185 @@ +tools: +- name: regex_find_replace + owner: galaxyp + tool_panel_section_label: Text Manipulation + tool_shed_url: toolshed.g2.bx.psu.edu +- name: fastqc + owner: devteam + tool_panel_section_label: Biotechnologie_M2 + tool_shed_url: toolshed.g2.bx.psu.edu +- name: fasta_compute_length + owner: devteam + tool_panel_section_label: Biotechnologie_M2 + tool_shed_url: toolshed.g2.bx.psu.edu +- name: small_rna_maps + owner: artbio + tool_panel_section_label: Biotechnologie_M2 + tool_shed_url: toolshed.g2.bx.psu.edu +- name: yac_clipper + owner: artbio + tool_panel_section_label: Biotechnologie_M2 + tool_shed_url: toolshed.g2.bx.psu.edu +- name: sr_bowtie_dataset_annotation + owner: artbio + tool_panel_section_label: Biotechnologie_M2 + tool_shed_url: toolshed.g2.bx.psu.edu +- name: sr_bowtie + owner: artbio + tool_panel_section_label: Biotechnologie_M2 + tool_shed_url: toolshed.g2.bx.psu.edu +- name: sambamba + owner: artbio + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_view + owner: iuc + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_stats + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_split + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_sort + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_slice_bam + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_rmdup + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_reheader + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_mpileup + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_idxstats + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_flagstat + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_calmd + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_bedcov + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: sam_to_bam + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: bam_to_sam + owner: devteam + tool_panel_section_label: SAMTOOLS + tool_shed_url: toolshed.g2.bx.psu.edu +- name: multiqc + owner: iuc + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: nextclade + owner: iuc + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: pangolin + owner: iuc + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: snpeff_sars_cov_2 + owner: iuc + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: ivar_consensus + owner: iuc + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: ivar_variants + owner: iuc + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: ivar_trim + owner: iuc + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: qualimap_bamqc + owner: iuc + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: samtools_view + owner: iuc + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: bwa + owner: devteam + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: text_processing + owner: bgruening + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: fastp + owner: iuc + tool_panel_section_label: COVID + tool_shed_url: toolshed.g2.bx.psu.edu +- name: cherry_pick_fasta + owner: artbio + tool_panel_section_label: Fasta Manipulation + tool_shed_url: toolshed.g2.bx.psu.edu +- name: lumpy_smoove + owner: artbio + tool_panel_section_label: Variants + tool_shed_url: toolshed.g2.bx.psu.edu +- name: repenrich + owner: artbio + tool_panel_section_label: Repenrich + tool_shed_url: toolshed.g2.bx.psu.edu +- name: bowtie2 + owner: devteam + tool_panel_section_label: 'NGS: align' + tool_shed_url: toolshed.g2.bx.psu.edu +- name: bowtie_wrappers + owner: devteam + tool_panel_section_label: 'NGS: align' + tool_shed_url: toolshed.g2.bx.psu.edu +- name: mapping_quality_stats + owner: artbio + tool_panel_section_label: 'NGS: QC and manipulation' + tool_shed_url: toolshed.g2.bx.psu.edu +- name: seqtk + owner: iuc + tool_panel_section_label: 'NGS: QC and manipulation' + tool_shed_url: toolshed.g2.bx.psu.edu +- name: data_manager_sam_fasta_index_builder + owner: devteam + tool_panel_section_label: None + tool_shed_url: toolshed.g2.bx.psu.edu +- name: data_manager_bowtie_index_builder + owner: iuc + tool_panel_section_label: None + tool_shed_url: toolshed.g2.bx.psu.edu +- name: data_manager_fetch_genome_dbkeys_all_fasta + owner: devteam + tool_panel_section_label: None + tool_shed_url: toolshed.g2.bx.psu.edu +- name: data_manager_bwa_mem_index_builder + owner: devteam + tool_panel_section_label: None + tool_shed_url: toolshed.g2.bx.psu.edu +- name: data_manager_bowtie2_index_builder + owner: devteam + tool_panel_section_label: None + tool_shed_url: toolshed.g2.bx.psu.edu diff --git a/environments/dev_mississippi/group_vars/all/galaxy b/environments/dev_mississippi/group_vars/all/galaxy index b7658f7..cb115c5 100644 --- a/environments/dev_mississippi/group_vars/all/galaxy +++ b/environments/dev_mississippi/group_vars/all/galaxy @@ -52,6 +52,11 @@ galaxy_config: # Automation / Ease of Use / User-facing features require_login: true + # settings in test before generalisation + user_activation_on: true + interactivetools_enable: true + allow_user_creation: false + enable_tool_source_display: true gravity: gunicorn: diff --git a/requirements.yml b/requirements.yml index 4aa2d56..6aa9efa 100644 --- a/requirements.yml +++ b/requirements.yml @@ -3,7 +3,7 @@ - src: galaxyproject.nginx version: 0.7.1 - src: galaxyproject.postgresql - version: 1.1.3 + version: 1.1.5 - src: galaxyproject.postgresql_objects version: 1.2.0 - src: galaxyproject.miniconda diff --git a/scripts/manage_bootstrap_user.py b/scripts/manage_bootstrap_user.py deleted file mode 100644 index c0b2b2b..0000000 --- a/scripts/manage_bootstrap_user.py +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env python - -import configparser -import argparse -import logging -import os -import re -import sys - -new_path = [os.path.join(os.getcwd(), "lib")] -new_path.extend(sys.path[1:]) -sys.path = new_path - -import galaxy.config -import sqlalchemy -import mercurial -from galaxy.model import mapping - -import yaml -from galaxy.security.idencoding import IdEncodingHelper as Security - -logging.captureWarnings(True) - -VALID_PUBLICNAME_RE = re.compile("^[a-z0-9\-]+$") -VALID_EMAIL_RE = re.compile("[^@]+@[^@]+\.[^@]+") - - -class BootstrapGalaxyApplication(object): - """ - Creates a basic Tool Shed application in order to discover the database - connection and use SQL to create a user and API key. - """ - - def __init__(self, config): - self.config = config - if not self.config.database_connection: - self.config.database_connection = \ - "sqlite:///%s?isolation_level=IMMEDIATE" % str(config.database) - # Setup the database engine and ORM - self.model = mapping.init(self.config.file_path, - self.config.database_connection, - engine_options={}, - create_tables=False) - self.security = Security(id_secret=self.config.id_secret) - - @property - def sa_session(self): - """Returns a SQLAlchemy session.""" - return self.model.context.current - - def shutdown(self): - pass - - -class ProgressConsoleHandler(logging.StreamHandler): - """ - A handler class which allows the cursor to stay on - one line for selected messages - """ - on_same_line = False - - def emit(self, record): - try: - msg = self.format(record) - stream = self.stream - same_line = hasattr(record, 'same_line') - if self.on_same_line and not same_line: - stream.write('\r\n') - stream.write(msg) - if same_line: - stream.write('.') - self.on_same_line = True - else: - stream.write('\r\n') - self.on_same_line = False - self.flush() - except (KeyboardInterrupt, SystemExit): - raise - except: - self.handleError(record) - - -def _setup_global_logger(): - formatter = logging.Formatter('%(asctime)s %(levelname)-5s - %(message)s') - file_handler = logging.FileHandler('/tmp/galaxy_tools_bootstrap_user.log') - progress = ProgressConsoleHandler() - console = logging.StreamHandler() - console.setFormatter(formatter) - - logger = logging.getLogger('test') - logger.setLevel(logging.DEBUG) - logger.addHandler(progress) - logger.addHandler(file_handler) - return logger - - -def create_api_key(app, user, preset_api_key=None): - """ - Creates a random new api key if preset_api_key is False, - otherwise sets the API key to preset_api_key. - """ - if preset_api_key: - api_key = preset_api_key - else: - api_key = app.security.get_new_guid() - new_key = app.model.APIKeys() - new_key.user_id = user.id - new_key.key = api_key - app.sa_session.add(new_key) - app.sa_session.flush() - return api_key - - -def get_or_create_api_key(app, user, preset_api_key=None): - if user.api_keys: - key = user.api_keys[0].key - else: - key = create_api_key(app, user, preset_api_key) - return key - - -def create_user(app, email, password, username): - if email and password and username: - invalid_message = validate(email, password, username) - if invalid_message: - log.error(invalid_message) - else: - user = app.model.User(email=email) - user.set_password_cleartext(password) - user.username = username - app.sa_session.add(user) - app.sa_session.flush() - app.model.security_agent.create_private_user_role(user) - return user - else: - log.error("Missing required values for email: {0}, password: {1}, " - "username: {2}".format(email, password, username)) - return None - - -def get_or_create_user(app, email, password, username): - user = app.sa_session.query(app.model.User).filter( - app.model.User.table.c.username == username).first() - if user: - return user - else: - return create_user(app, email, password, username) - - -def delete_user(app, username): - user = app.sa_session.query(app.model.User).filter( - app.model.User.table.c.username == username).first() - app.sa_session.delete(user) - app.sa_session.flush() - return user - - -def validate(email, password, username): - message = validate_email(email) - if not message: - message = validate_password(password) - if not message: - message = validate_publicname(username) - return message - - -def validate_email(email): - """Validates the email format.""" - message = '' - if not(VALID_EMAIL_RE.match(email)): - message = "Please enter a real email address." - elif len(email) > 255: - message = "Email address exceeds maximum allowable length." - return message - - -def validate_password(password): - if len(password) < 5: - return "Use a password of at least 5 characters" - return '' - - -def validate_publicname(username): - """Validates the public username.""" - if len(username) < 3: - return "Public name must be at least 3 characters in length" - if len(username) > 255: - return "Public name cannot be more than 255 characters in length" - if not(VALID_PUBLICNAME_RE.match(username)): - return "Public name must contain only lower-case letters, \ - numbers and '-'" - return '' - - -def get_bootstrap_app(ini_file): - config_parser = configparser.ConfigParser({'here': os.getcwd()}) - with open(ini_file, 'r') as ymlfile: - cfg = yaml.full_load(ymlfile) - config_dict = cfg['galaxy'] - config = galaxy.config.Configuration(**config_dict) - app = BootstrapGalaxyApplication(config) - return app - - -def create_bootstrap_user(ini_file, - username, - user_email, - password, - preset_api_key=None): - app = get_bootstrap_app(ini_file) - user = get_or_create_user(app, user_email, password, username) - if user is not None: - api_key = get_or_create_api_key(app, user, preset_api_key) - print(api_key) - exit(0) - else: - log.error("Problem creating a new user: {0} and an associated API key." - .format(username)) - exit(1) - - -def delete_bootstrap_user(ini_file, username): - app = get_bootstrap_app(ini_file) - user = delete_user(app, username) - if user is not None: - exit(0) - else: - log.error("Problem deleting user: {0}".format(username)) - exit(1) - - -if __name__ == "__main__": - global log - log = _setup_global_logger() - parser = argparse.ArgumentParser( - prog = "manage_bootstrap_user.py", - description = "usage: python %prog [options]", - epilog = "Be sure to execute in root galaxy dir and galaxy venv") - parser.add_argument("-c", "--config", - required=True, - help="Path to ") - subparsers = parser.add_subparsers( - title="action", help='create or delete bootstrap users') - parser_create = subparsers.add_parser('create', - help='create a new bootstrap user') - parser_create.set_defaults(action='create') - parser_create.add_argument("-u", "--username", - default="cloud", - help="Username to create. Defaults to 'cloud'", - required=True) - parser_create.add_argument("-e", "--email", - default="cloud@galaxyproject.org", - help="Email for user", - required=True) - parser_create.add_argument("-p", "--password", - default="password", - help="Password for user", - required=True) - parser_create.add_argument("-a", "--preset_api_key", - default=None, - help="Set API key for user", - required=False) - parser_delete = subparsers.add_parser( - 'delete', help='delete an existing bootstrap user') - parser_delete.set_defaults(action='delete') - parser_delete.add_argument("-u", "--username", - default="cloud", - required=True, - help="Username to delete. Defaults to 'cloud'",) - args = parser.parse_args() - - if args.action == "create": - create_bootstrap_user(args.config, - args.username, - args.email, - args.password, - args.preset_api_key) - elif args.action == "delete": - delete_bootstrap_user(args.config, args.username)