diff --git a/ansible.cfg b/ansible.cfg
index bc40777..1a43c9d 100644
--- a/ansible.cfg
+++ b/ansible.cfg
@@ -1,14 +1,16 @@
[defaults]
interpreter_python = /usr/bin/python3
inventory = hosts
+bin_ansible_callbacks = True
retry_files_enabled = false
vault_password_file = .vault-password.txt
gather_subset=!hardware
-no_log = True
+#no_log = True
+log_path = /var/log/ansible.log
# Show diffs of changes
[diff]
-always = False
+always = True
[ssh_connection]
-pipelining = true
+pipelining = True
diff --git a/galaxy.yml b/galaxy.yml
index 5b9816d..304e356 100644
--- a/galaxy.yml
+++ b/galaxy.yml
@@ -5,12 +5,46 @@
pre_tasks:
- name: Install Dependencies
package:
- name: 'acl'
+ name: ['acl', 'logrotate']
roles:
- galaxyproject.postgresql
- role: galaxyproject.postgresql_objects
become: true
become_user: postgres
+ post_tasks:
+ - name: Create pg_dump backup folder if needed
+ file:
+ path: "{{ postgresql_backup_dir }}/pg_dump"
+ state: directory
+ owner: postgres
+ group: "{{ galaxy_user_group }}"
+ mode: 0750
+ - name: Create initial empty pg_dump file if needed # otherwise the following logrotate conf will not work
+ file:
+ path: "{{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql.gz"
+ state: touch
+ owner: postgres
+ group: "{{ galaxy_user_group }}"
+ mode: 0640
+ - name: Set-up Galaxy DB backup using pg_dump and logrotate
+ ansible.builtin.blockinfile:
+ path: "/etc/logrotate.d/{{ postgresql_pg_dump_prefix }}"
+ create: true
+ block: |
+ {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql.gz {
+ daily
+ missingok
+ rotate 14
+ nocompress
+ nocreate
+ postrotate
+ echo "### Start backup at `date` ###" >> /var/log/{{ postgresql_pg_dump_prefix }}_errors.log
+ sudo -u postgres pg_dump {{ galaxy_db_name }} --file={{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql --verbose >> /var/log/{{ postgresql_pg_dump_prefix }}_errors.log 2>&1
+ sudo chmod 640 {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql >> /var/log/{{ postgresql_pg_dump_prefix }}_errors.log 2>&1
+ sudo -u postgres gzip {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql >> /var/log/{{ postgresql_pg_dump_prefix }}_errors.log 2>&1
+ echo "### Finished backup at `date` ###" >> /var/log/{{ postgresql_pg_dump_prefix }}_errors.log
+ endscript
+ }
- hosts: galaxyservers
become: true
@@ -18,7 +52,11 @@
vars_files:
- group_vars/secret.yml
pre_tasks:
- - pip:
+ - name: Install Dependencies
+ ansible.builtin.package:
+ name: ['acl', 'bzip2', 'git', 'make', 'tar', 'python3-bioblend', 'python3-venv', 'python3-setuptools', 'python3-pip', 'python3-psycopg2', 'rsync', 'python3-docker', 'nginx-full']
+ - name: Install additional pip packages
+ ansible.builtin.pip:
name: passlib
- htpasswd:
path: /etc/nginx/passwd
@@ -27,9 +65,6 @@
owner: www-data # nginx on centos
group: root
mode: 0640
- - name: Install Dependencies
- package:
- name: ['acl', 'bzip2', 'git', 'make', 'tar', 'python3-bioblend', 'python3-venv', 'python3-setuptools']
- name: Install RHEL/CentOS/Rocky specific dependencies
package:
name: ['tmpwatch']
@@ -38,30 +73,57 @@
package:
name: ['tmpreaper']
when: ansible_os_family == 'Debian'
+ - name: Ensure the python command exists
+ ansible.builtin.shell:
+ cmd: which python||exit 0
+ register: python_path
+ - name: Register python3 path
+ ansible.builtin.command: which python3
+ register: python3_path
+ when: python_path.stdout == ""
+ - name: Link python to python3 if missing
+ ansible.builtin.file:
+ src: python3_path.stdout
+ dest: python3_path.stdout[:-1]
+ state: link
+ force: yes
+ when: python_path.stdout == ""
- git:
repo: 'https://github.com/usegalaxy-eu/libraries-training-repo'
dest: /libraries/
+ when: "'usegalaxy' in inventory_hostname"
+# - name: Create data storage for UMSA
+# file:
+# owner: "{{ galaxy_user_name }}"
+# group: "{{ galaxy_user_group }}"
+# path: "{{ galaxy_data_prefix }}"
+# state: directory
+# mode: "0755"
+# when: "'umsa' in inventory_hostname"
- name: Create the RBD storage directory
file:
- owner: galaxy
- group: galaxy
+ owner: "{{ galaxy_user_name }}"
+ group: "{{ galaxy_user_group }}"
path: /rbd/data
state: directory
mode: '0755'
+ when: "'usegalaxy' in inventory_hostname"
- name: Create the nginx upload job files store
file:
owner: www-data
- group: galaxy
+ group: "{{ galaxy_user_group }}"
path: /rbd/data/_upload_job_files
state: directory
mode: '0775'
+ when: "'usegalaxy' in inventory_hostname"
- name: Create the client body temp path
file:
owner: www-data
- group: galaxy
+ group: "{{ galaxy_user_group }}"
path: /rbd/data/nginx_upload_store
state: directory
mode: '0775'
+ when: "'usegalaxy' in inventory_hostname"
roles:
- galaxyproject.tusd
- usegalaxy_eu.apptainer
@@ -76,15 +138,15 @@
- galaxyproject.proftpd
- geerlingguy.docker
- usegalaxy_eu.rabbitmqserver
- - galaxyproject.tiaas2
+# - galaxyproject.tiaas2
- galaxyproject.gxadmin
- galaxyproject.cvmfs
- - dj-wasabi.telegraf
+# - dj-wasabi.telegraf
post_tasks:
- name: Setup gxadmin cleanup task
ansible.builtin.cron:
name: "Cleanup Old User Data"
- user: galaxy # Run as the Galaxy user
+ user: "{{ galaxy_user_name }}" # Run as the Galaxy user
minute: "0"
hour: "0"
job: "GALAXY_LOG_DIR=/tmp/gxadmin/ GALAXY_ROOT={{ galaxy_root }}/server /usr/local/bin/gxadmin galaxy cleanup 60"
@@ -95,3 +157,11 @@
minute: "0"
hour: "*"
job: "python3 {{ playbook_dir }}/add_group_users.py {{ inventory_hostname }} {{ api_key }}"
+ when: "'usegalaxy' in inventory_hostname"
+ - name: RE Galaxy instance patch in lib/galaxy/webapps/galaxy/api/job_files.py
+ replace:
+ path: "{{ galaxy_root }}/server/lib/galaxy/webapps/galaxy/api/job_files.py"
+ regexp: '^(\s*?)shutil\.move\(input_file\.name,\s*path\)\s*$'
+ replace: '\1os.chmod(shutil.move(input_file.name, path), 0o664)'
+ validate: 'python -m py_compile %s'
+ when: "'galaxy-re' in inventory_hostname or 'galaxy-re' in ansible_hostname"
diff --git a/group_vars/all.yml b/group_vars/all.yml
index a36f40d..6b084d5 100644
--- a/group_vars/all.yml
+++ b/group_vars/all.yml
@@ -3,8 +3,11 @@
pip_virtualenv_command: /usr/bin/python3 -m venv # usegalaxy_eu.certbot, usegalaxy_eu.tiaas2, galaxyproject.galaxy
# Common variables needed by all hosts
-galaxy_user_name: galaxy
+galaxy_user_name: galaxyumsa
+galaxy_user_id: 10456
+galaxy_user_group: galaxyumsa
galaxy_db_name: galaxy
+galaxy_data_prefix: "/mnt/data"
# CVMFS vars
cvmfs_role: client
diff --git a/group_vars/dbservers.yml b/group_vars/dbservers.yml
index 2504faf..0082b0f 100644
--- a/group_vars/dbservers.yml
+++ b/group_vars/dbservers.yml
@@ -13,28 +13,28 @@ postgresql_objects_privileges:
privs: SELECT
objs: ALL_IN_SCHEMA
- - database: galaxy
- roles: tiaas
- objs: galaxy_user,galaxy_session,job,history,workflow,workflow_invocation
- type: table
- privs: SELECT
- - database: galaxy
- roles: tiaas
- objs: user_group_association,galaxy_group,role,group_role_association
- type: table
- privs: SELECT,INSERT
- - database: galaxy
- roles: tiaas
- objs: group_role_association
- type: table
- privs: DELETE
- - database: galaxy
- roles: tiaas
- objs: role_id_seq,galaxy_group_id_seq,group_role_association_id_seq,user_group_association_id_seq
- type: sequence
- privs: USAGE,SELECT
+# - database: galaxy
+# roles: tiaas
+# objs: galaxy_user,galaxy_session,job,history,workflow,workflow_invocation
+# type: table
+# privs: SELECT
+# - database: galaxy
+# roles: tiaas
+# objs: user_group_association,galaxy_group,role,group_role_association
+# type: table
+# privs: SELECT,INSERT
+# - database: galaxy
+# roles: tiaas
+# objs: group_role_association
+# type: table
+# privs: DELETE
+# - database: galaxy
+# roles: tiaas
+# objs: role_id_seq,galaxy_group_id_seq,group_role_association_id_seq,user_group_association_id_seq
+# type: sequence
+# privs: USAGE,SELECT
# PostgreSQL Backups
-postgresql_backup_dir: /data/backups
-postgresql_backup_local_dir: "{{ '~postgres' | expanduser }}/backups"
-
+postgresql_backup_dir: "{{ galaxy_data_prefix }}/backups"
+postgresql_backup_local_dir: "{{ galaxy_data_prefix }}/backups/scripts"
+postgresql_pg_dump_prefix: galaxy_db_backup
diff --git a/group_vars/galaxyservers.yml b/group_vars/galaxyservers.yml
index b657dbb..35d031d 100644
--- a/group_vars/galaxyservers.yml
+++ b/group_vars/galaxyservers.yml
@@ -6,13 +6,19 @@ galaxy_manage_paths: true # False by default as your administrator might e.g. ha
galaxy_manage_cleanup: true
galaxy_layout: root-dir
galaxy_root: /srv/galaxy
-galaxy_user: {name: "{{ galaxy_user_name }}", shell: /bin/bash}
- # galaxy_commit_id: 1ad49865fbeb03551cf7774dc0c12e5cd27ff797 # release_23.0
+galaxy_tool_dependency_dir: "{{ galaxy_mutable_data_dir }}/dependencies"
+galaxy_user:
+ name: "{{ galaxy_user_name }}"
+ uid: "{{ galaxy_user_id }}"
+ group: "{{ galaxy_user_group }}"
+ shell: /bin/bash
+# galaxy_commit_id: 1ad49865fbeb03551cf7774dc0c12e5cd27ff797 # release_23.0
galaxy_commit_id: 72070dbd0a6d0f418d0b8f914825ea2b8e6a0a88
galaxy_force_checkout: true
miniconda_prefix: "{{ galaxy_tool_dependency_dir }}/_conda"
-miniconda_version: 23.5.2 # 23.5.0 # 4.12.0
-miniconda_channels: ['conda-forge', 'defaults']
+miniconda_version: 'latest' # 23.5.2 # 23.5.0 # 4.12.0
+miniconda_channels: ['conda-forge', 'bioconda', 'anaconda', 'defaults']
+miniconda_base_env_packages: ['mamba']
# Galaxy Job Configuration
galaxy_job_config:
@@ -20,30 +26,30 @@ galaxy_job_config:
local_runner:
load: galaxy.jobs.runners.local:LocalJobRunner
workers: 4
- pulsar_runner:
- load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner
- galaxy_url: "https://{{ rabbitmq_hostname }}"
- amqp_url: "pyamqp://pulsar:{{ rabbitmq_users_password.pulsar }}@{{ rabbitmq_hostname }}:5671/pulsar?ssl=1"
- amqp_acknowledge: true
- amqp_ack_republish_time: 1200
- amqp_consumer_timeout: 2
- amqp_publish_retry: true
- amqp_publish_retry_max_retries: 60
- manager: _default_
- pulsar_gpu_runner:
- load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner
- galaxy_url: "https://{{ rabbitmq_hostname }}"
- amqp_url: "pyamqp://pulsar:{{ rabbitmq_users_password.pulsar }}@{{ rabbitmq_hostname }}:5671/pulsar?ssl=1"
- amqp_acknowledge: true
- amqp_ack_republish_time: 1200
- amqp_consumer_timeout: 2
- amqp_publish_retry: true
- amqp_publish_retry_max_retries: 60
- manager: gpu
+ #pulsar_runner:
+ # load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner
+ # galaxy_url: "https://{{ rabbitmq_hostname }}"
+ # amqp_url: "pyamqp://pulsar:{{ vault_rabbitmq_password.pulsar }}@{{ rabbitmq_hostname }}:5671/pulsar?ssl=1"
+ # amqp_acknowledge: true
+ # amqp_ack_republish_time: 1200
+ # amqp_consumer_timeout: 2
+ # amqp_publish_retry: true
+ # amqp_publish_retry_max_retries: 60
+ # manager: _default_
+ #pulsar_gpu_runner:
+ # load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner
+ # galaxy_url: "https://{{ rabbitmq_hostname }}"
+ # amqp_url: "pyamqp://pulsar:{{ vault_rabbitmq_password.pulsar }}@{{ rabbitmq_hostname }}:5671/pulsar?ssl=1"
+ # amqp_acknowledge: true
+ # amqp_ack_republish_time: 1200
+ # amqp_consumer_timeout: 2
+ # amqp_publish_retry: true
+ # amqp_publish_retry_max_retries: 60
+ # manager: gpu
# pulsar_runner:
# load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner
# galaxy_url: "https://{{ inventory_hostname }}"
- # amqp_url: "pyamqp://pulsar:{{ rabbitmq_users_password.galaxy }}@{{ inventory_hostname }}:5671/pulsar?ssl=1"
+ # amqp_url: "pyamqp://pulsar:{{ vault_rabbitmq_password.galaxy }}@{{ inventory_hostname }}:5671/pulsar?ssl=1"
# amqp_acknowledge: true
# amqp_ack_republish_time: 1200
# amqp_consumer_timeout: 2
@@ -53,7 +59,8 @@ galaxy_job_config:
handling:
assign: ['db-skip-locked']
execution:
- default: tpv_dispatcher
+ default: local_env
+ #default: tpv_dispatcher
environments:
local_env:
runner: local_runner
@@ -71,71 +78,71 @@ galaxy_job_config:
# Apptainer uses a temporary directory to build the squashfs filesystem
- name: APPTAINER_TMPDIR
value: /tmp
- pulsar:
- runner: pulsar_runner
- default_file_action: remote_transfer
- dependency_resolution: remote
- jobs_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-cz/files/staging"
- persistence_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-cz/files/persistent"
- remote_metadata: false
- rewrite_parameters: true
- transport: curl
- outputs_to_working_directory: false
- # submit_native_specification: '-l select=1:ncpus=2:mem=8gb:scratch_local=50gb -l walltime=12:00:00 -q galaxyeu@elixir-pbs.elixir-czech.cz'
- singularity_enabled: true
- singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro,$SCRATCHDIR"
- ## Following configuration works!
- # singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro"
- container_resolvers:
- - type: explicit_singularity
- - type: mulled_singularity
- env:
- # Ensuring a consistent collation environment is good for reproducibility.
- - name: LC_ALL
- value: C
- - name: TMPDIR
- value: "$SCRATCHDIR"
- - name: TMP
- value: "$SCRATCHDIR"
- - name: TEMP
- value: "$SCRATCHDIR"
- pulsar_gpu:
- runner: pulsar_gpu_runner
- default_file_action: remote_transfer
- transport: curl
- dependency_resolution: remote
- jobs_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-cz/files/staging"
- persistence_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-cz/files/persistent"
- remote_metadata: false
- rewrite_parameters: true
- outputs_to_working_directory: false
- singularity_enabled: true
- singularity_run_extra_arguments: '--nv'
- singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,$ALPHAFOLD_DB:/data:ro'
- env:
- # The cache directory holds the docker containers that get converted
- - name: SINGULARITY_CACHEDIR
- value: "/storage/praha5-elixir/home/galaxyeu/singularity/cache"
- - name: APPTAINER_CACHEDIR
- value: "/storage/praha5-elixir/home/galaxyeu/singularity/cache"
- # Singularity uses a temporary directory to build the squashfs filesystem
- - name: SINGULARITY_TMPDIR
- value: "/storage/praha5-elixir/home/galaxyeu/singularity/tmp"
- - name: APPTAINER_TMPDIR
- value: "/storage/praha5-elixir/home/galaxyeu/singularity/tmp"
- # Alphafold specific variables
- - name: ALPHAFOLD_DB
- value: "/storage/brno11-elixir/projects/alphafold/alphafold.db-2.3.1"
- # Default variables
- - name: TMPDIR
- value: "$SCRATCHDIR"
- - name: TMP
- value: "$SCRATCHDIR"
- - name: TEMP
- value: "$SCRATCHDIR"
- # Ensuring a consistent collation environment is good for reproducibility.
- - name: LC_ALL
- value: C
+# pulsar:
+# runner: pulsar_runner
+# default_file_action: remote_transfer
+# dependency_resolution: remote
+# jobs_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-cz/files/staging"
+# persistence_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-cz/files/persistent"
+# remote_metadata: false
+# rewrite_parameters: true
+# transport: curl
+# outputs_to_working_directory: false
+# # submit_native_specification: '-l select=1:ncpus=2:mem=8gb:scratch_local=50gb -l walltime=12:00:00 -q galaxyeu@elixir-pbs.elixir-czech.cz'
+# singularity_enabled: true
+# singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro,$SCRATCHDIR"
+# ## Following configuration works!
+# # singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro"
+# container_resolvers:
+# - type: explicit_singularity
+# - type: mulled_singularity
+# env:
+# # Ensuring a consistent collation environment is good for reproducibility.
+# - name: LC_ALL
+# value: C
+# - name: TMPDIR
+# value: "$SCRATCHDIR"
+# - name: TMP
+# value: "$SCRATCHDIR"
+# - name: TEMP
+# value: "$SCRATCHDIR"
+# pulsar_gpu:
+# runner: pulsar_gpu_runner
+# default_file_action: remote_transfer
+# transport: curl
+# dependency_resolution: remote
+# jobs_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-cz/files/staging"
+# persistence_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-cz/files/persistent"
+# remote_metadata: false
+# rewrite_parameters: true
+# outputs_to_working_directory: false
+# singularity_enabled: true
+# singularity_run_extra_arguments: '--nv'
+# singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,$ALPHAFOLD_DB:/data:ro'
+# env:
+# # The cache directory holds the docker containers that get converted
+# - name: SINGULARITY_CACHEDIR
+# value: "/storage/praha5-elixir/home/galaxyeu/singularity/cache"
+# - name: APPTAINER_CACHEDIR
+# value: "/storage/praha5-elixir/home/galaxyeu/singularity/cache"
+# # Singularity uses a temporary directory to build the squashfs filesystem
+# - name: SINGULARITY_TMPDIR
+# value: "/storage/praha5-elixir/home/galaxyeu/singularity/tmp"
+# - name: APPTAINER_TMPDIR
+# value: "/storage/praha5-elixir/home/galaxyeu/singularity/tmp"
+# # Alphafold specific variables
+# - name: ALPHAFOLD_DB
+# value: "/storage/brno11-elixir/projects/alphafold/alphafold.db-2.3.1"
+# # Default variables
+# - name: TMPDIR
+# value: "$SCRATCHDIR"
+# - name: TMP
+# value: "$SCRATCHDIR"
+# - name: TEMP
+# value: "$SCRATCHDIR"
+# # Ensuring a consistent collation environment is good for reproducibility.
+# - name: LC_ALL
+# value: C
tpv_dispatcher:
runner: dynamic
type: python
@@ -144,24 +151,18 @@ galaxy_job_config:
tpv_config_files:
- https://raw.githubusercontent.com/galaxyproject/tpv-shared-database/main/tools.yml
- "{{ tpv_config_dir }}/tpv_rules_local.yml"
- resources:
- default: default
- groups:
- default: []
- testing: [cores, time]
tools:
- class: local # these special tools that aren't parameterized for remote execution - expression tools, upload, etc
environment: local_env
- id: testing
environment: tpv_dispatcher
- resources: testing
galaxy_config:
galaxy:
# Main Configuration
# To not bother the main disk
- new_file_path: "/rbd/data/tmp"
- nginx_upload_job_files_store: "/rbd/data/_upload_job_files"
+ new_file_path: "{{ galaxy_data_prefix }}/tmp"
+ nginx_upload_job_files_store: "{{ galaxy_data_prefix }}/_upload_job_files"
nginx_upload_job_files_path: "/_job_files"
object_store_config_file: "{{ galaxy_config_dir }}/object_store_conf.xml"
enable_oidc: true
@@ -174,13 +175,12 @@ galaxy_config:
- 499542@muni.cz
- 325073@mail.muni.cz
- demko@cesnet.cz
- # cleanup_job: never
smtp_server: "rs.cesnet.cz:25"
error_email_to: "regalaxy@rt.cesnet.cz"
allow_user_creation: False
database_connection: "postgresql:///{{ galaxy_db_name }}?host=/var/run/postgresql"
- file_path: /mnt/data/datasets
- job_working_directory: /mnt/data/jobs
+ file_path: "{{ galaxy_data_prefix }}/datasets"
+ job_working_directory: "{{ galaxy_data_prefix }}/jobs"
object_store_store_by: uuid
id_secret: "{{ vault_id_secret }}"
job_config: "{{ galaxy_job_config }}" # Use the variable we defined above
@@ -217,18 +217,18 @@ galaxy_config:
dependency_resolvers_config_file: "{{ galaxy_config_dir }}/dependency_resolvers_conf.xml"
containers_resolvers_config_file: "{{ galaxy_config_dir }}/container_resolvers_conf.yml"
# Data Library Directories
- library_import_dir: /libraries/admin
- user_library_import_dir: /libraries/user
+# library_import_dir: /libraries/admin
+# user_library_import_dir: /libraries/user
# Celery
- amqp_internal_connection: "pyamqp://galaxy:{{ vault_rabbitmq_password_galaxy }}@localhost:5671/galaxy_internal?ssl=1"
+ amqp_internal_connection: "pyamqp://galaxy:{{ vault_rabbitmq_password.galaxy }}@localhost:5671/galaxy_internal?ssl=1"
celery_conf:
result_backend: "redis://localhost:6379/0"
enable_celery_tasks: true
# Monitoring
- statsd_host: localhost
- statsd_influxdb: true
+# statsd_host: localhost
+# statsd_influxdb: true
# FTP
- ftp_upload_dir: /mnt/data/uploads
+ ftp_upload_dir: "{{ galaxy_data_prefix}}/uploads"
ftp_upload_site: "{{ inventory_hostname }}"
gravity:
process_manager: systemd
@@ -265,21 +265,21 @@ galaxy_config:
pools:
- job-handlers
- workflow-schedulers
- reports:
- enable: true
- url_prefix: /reports
- bind: "unix:{{ galaxy_mutable_config_dir }}/reports.sock"
- config_file: "{{ galaxy_config_dir }}/reports.yml"
+# reports:
+# enable: false
+# url_prefix: /reports
+# bind: "unix:{{ galaxy_mutable_config_dir }}/reports.sock"
+# config_file: "{{ galaxy_config_dir }}/reports.yml"
galaxy_job_config_file: "{{ galaxy_config_dir }}/galaxy.yml"
-galaxy_config_files_public:
- - src: files/galaxy/welcome.html
- dest: "{{ galaxy_mutable_config_dir }}/welcome.html"
+#galaxy_config_files_public:
+# - src: files/galaxy/welcome.html
+# dest: "{{ galaxy_mutable_config_dir }}/welcome.html"
galaxy_config_files:
- - src: files/galaxy/themes.yml
- dest: "{{ galaxy_config.galaxy.themes_config_file }}"
+# - src: files/galaxy/themes.yml
+# dest: "{{ galaxy_config.galaxy.themes_config_file }}"
- src: files/galaxy/config/tpv_rules_local.yml
dest: "{{ tpv_mutable_dir }}/tpv_rules_local.yml"
@@ -290,8 +290,10 @@ galaxy_config_templates:
dest: "{{ galaxy_config.galaxy.dependency_resolvers_config_file }}"
- src: templates/galaxy/config/job_resource_params_conf.xml.j2
dest: "{{ galaxy_config.galaxy.job_resource_params_file }}"
- - src: templates/galaxy/config/reports.yml
- dest: "{{ galaxy_config.gravity.reports.config_file }}"
+ - src: templates/galaxy/config/sanitize_allowlist.txt
+ dest: "{{ galaxy_config.galaxy.sanitize_allowlist_file }}"
+# - src: templates/galaxy/config/reports.yml
+# dest: "{{ galaxy_config.gravity.reports.config_file }}"
- src: templates/galaxy/config/oidc_config.xml
dest: "{{ galaxy_config_dir }}/oidc_config.xml"
- src: templates/galaxy/config/oidc_backends_config.xml
@@ -300,7 +302,7 @@ galaxy_config_templates:
dest: "{{ galaxy_config.galaxy.object_store_config_file }}"
galaxy_extra_dirs:
- - /mnt/data
+ - "{{ galaxy_data_prefix }}"
- "{{ galaxy_config_dir }}/{{ tpv_config_dir_name }}"
galaxy_extra_privsep_dirs:
@@ -338,7 +340,7 @@ nginx_servers:
- redirect-ssl
nginx_ssl_servers:
- galaxy
- # - sentry
+# - sentry
nginx_enable_default_server: false
nginx_conf_http:
client_max_body_size: 1g
@@ -400,28 +402,27 @@ rabbitmq_config:
rabbitmq_vhosts:
- pulsar
- - galaxy_gpu
- galaxy_internal
rabbitmq_users:
- user: debian
- password: "{{ rabbitmq_users_password.mqadmin }}"
+ password: "{{ vault_rabbitmq_password.mqadmin }}"
tags: administrator
vhost: /
- user: pulsar
- password: "{{ rabbitmq_users_password.pulsar }}"
+ password: "{{ vault_rabbitmq_password.pulsar }}"
vhost: pulsar
- user: galaxy
- password: "{{ vault_rabbitmq_password_galaxy }}"
+ password: "{{ vault_rabbitmq_password.galaxy }}"
vhost: galaxy_internal
- user: flower
- password: "{{ vault_rabbitmq_password_flower }}"
+ password: "{{ vault_rabbitmq_password.flower }}"
tags: administrator
vhost: galaxy_internal
# TUS
galaxy_tusd_port: 1080
-galaxy_tus_upload_store: /rbd/data/tus # /mnt/data/tus
+galaxy_tus_upload_store: "{{ galaxy_data_prefix }}/tus" # /mnt/data/tus
#Redis
galaxy_additional_venv_packages:
@@ -429,44 +430,27 @@ galaxy_additional_venv_packages:
# Flower
flower_python_version: python3
+flower_bind_interface: localhost
+flower_port: 5555
flower_app_dir: "{{ galaxy_root }}"
flower_python_path: "{{ galaxy_root }}/server/lib"
flower_venv_dir: "{{ galaxy_venv_dir }}"
flower_app_name: galaxy.celery
flower_db_file: "{{ galaxy_root }}/var/flower.db"
flower_persistent: true
-flower_broker_api: "https://flower:{{ vault_rabbitmq_password_flower }}@localhost:5671/api/"
-flower_broker_url: "amqp://flower:{{ vault_rabbitmq_password_flower }}@localhost:5671/galaxy_internal?ssl=true"
+flower_broker_api: "https://flower:{{ vault_rabbitmq_password.flower }}@localhost:5671/api/"
+flower_broker_url: "pyamqp://flower:{{ vault_rabbitmq_password.flower }}@localhost:5671/galaxy_internal?ssl=1"
+#flower_broker_url: "pyamqp://flower:{{ rabbitmq_users_password.flower }}@{{ inventory_hostname }}:5671/galaxy_internal?ssl=1"
flower_proxy_prefix: /flower
-
+flower_user: "{{ galaxy_user.name }}"
+flower_group: "{{ galaxy_user.group }}"
flower_ui_users:
- name: admin
- password: "{{ vault_flower_user_password}}"
+ password: "{{ flower_admin_password }}"
flower_environment_variables:
GALAXY_CONFIG_FILE: "{{ galaxy_config_file }}"
-# Telegraf
-telegraf_plugins_extra:
- listen_galaxy_routes:
- plugin: "statsd"
- config:
- - service_address = ":8125"
- - metric_separator = "."
- - allowed_pending_messages = 10000
- monitor_galaxy_queue:
- plugin: "exec"
- config:
- - commands = ["/usr/bin/env PGDATABASE=galaxy /usr/local/bin/gxadmin iquery queue-overview --short-tool-id"]
- - timeout = "10s"
- - data_format = "influx"
- - interval = "15s"
-
-# TIaaS setup
-tiaas_dir: /srv/tiaas
-tiaas_admin_user: admin
-tiaas_admin_pass: changeme
-
# Proftpd:
proftpd_galaxy_auth: yes
galaxy_ftp_upload_dir: "{{ galaxy_config.galaxy.ftp_upload_dir }}"
@@ -476,15 +460,14 @@ proftpd_display_connect: |
Unauthorized access is prohibited
proftpd_create_ftp_upload_dir: yes
proftpd_options:
- - User: galaxy
- - Group: galaxy
+ - User: "{{ galaxy_user.name }}"
+ - Group: "{{ galaxy_user.group }}"
- Port: 21
-proftpd_sql_db: galaxy@/var/run/postgresql
-proftpd_sql_user: galaxy
+proftpd_sql_db: "{{ galaxy_db_name }}@/var/run/postgresql"
+proftpd_sql_user: "{{ galaxy_user.name }}"
proftpd_conf_ssl_certificate: /etc/letsencrypt/live/{{ inventory_hostname }}/cert.pem
proftpd_conf_ssl_certificate_key: /etc/letsencrypt/live/{{ inventory_hostname }}/privkey.pem
proftpd_global_options:
- PassivePorts: 56000 60000
proftpd_use_mod_tls_shmcache: false
proftpd_tls_options: NoSessionReuseRequired
-
diff --git a/group_vars/secret.yml b/group_vars/secret.yml
index eb3d58a..327f180 100644
--- a/group_vars/secret.yml
+++ b/group_vars/secret.yml
@@ -1,46 +1,30 @@
$ANSIBLE_VAULT;1.1;AES256
-30303337656261656337666563646337353937343463663462643364653435633864396434373730
-3861376165613462656536633532326331373265663836350a663963613461383734613038653462
-61646663323835363465333536323961613836306564623431303630326665313462633061336634
-6662346165646435300a373938366139356565626434633637616133313530613061363037343439
-38366234393537653534303038366264663336636336663038303564646163653864316235383263
-36613032646261366163313132316531396439393334653833366533353332363734303438653163
-63323464366335666432396634386231636662643661393233393564666135666634623065623832
-30393261666363323337336532656433383364396134616230346162303637363734323938316238
-33393665336562646138613165303562646463623033343631663065343735616536313638303830
-62376462303765346431643333666662333131393236363066376661626139343061303336653338
-34663264633163376230646139343635323439656439313363346565333934343831653430623834
-34663266633436616432383838383963383032393765663764383736366566313864333337393637
-63333366373130653837633465643839653063346262373631333666626538323662616565303363
-63633163353362626662383035643939336465633239643730313162636330613663633564636630
-31643534303934656263623335613366386136346161393835633632613166346635353862633466
-63306166633762623665353934353362656162323835653865626333353462326466376135306264
-39616335343163653366633562333631336364316230336432353763363236303764653937346432
-34366532313631346331393464323633393338626363353637316238353262633933333030326264
-30626665316630353866386232663038333237643231303033636530303361343764356232353134
-39376138336562373233353962666264336630363432396561313665376533663262653765383064
-32653862623966396238633062616130663563326661636137333437653862386564313536643531
-35343838313135363435323665343534343633316635323564633336393633346235366263396138
-31313835623262313661306337326665373666343336633934656162356431363836343131646330
-37313066353261656633613430373538656331653161343866373235646338343338376465396431
-30363031353062646365313066643938663030613264313561316562383432373165333862633766
-37316566643930373464303634396439353963613062393335643466656538663238386634336437
-32343363313434643031653732343661343566343666393032353533666462303765656135343531
-37393130306561313737366331313134343130373162653736653631353835366564356233313165
-30343965643536333462616566333162303135336566643337386636656665343462373533396662
-30633866653633363461343136313038326337616139636230326331323036316661366566656431
-35333835386364353363623337323334303238383264383039336330306635323564353264666365
-62613036346637623631386630643662626164656531376665393965666535663261333939636366
-37313163323935626363613334323939653331616664383061663965376366343563323735646430
-36663934353866323632366638373564333065653434316161376533656365653464333739313766
-38623335373337656261336535393032633932613231336163313033336537313434313336643763
-34353262663265363364626465663532303630363761653463373838356532336432356563343239
-34636436623539376565616165386432656638323964316539626564623433633563616634346564
-65346163323633363335316338663335663735663830653965306364663662616331383739313564
-61336662323534666465373233613933303934343738626466613535383461646163623630373430
-64653561363536366630653831633431613863646361353365316235393963663533636534326466
-31616230393936646635303235656563626531393235636262363063616461653963303632646136
-33326461323861646137616537623162653034656537333261366466356439313963623637346465
-31323938363164656261316538613538643733303439653662646637343964386561396337343533
-39363737653738663435646134653936363737306431323733376163633162393065313137666533
-32653763666264653263376563623866373635656436373364353336323531326432
+64626132343163623330373930356561326465323738616538373763313731343730343262383337
+6531346361343038376438313562623430336138396530320a363866643862333161356436343935
+37366533373035323065633962386435343338616135313834303336633361383864383831653237
+3536383037636636390a303437643233663166633131353038663937646439323035616134356236
+33663830346664323163303863633066393539383338653639356164643034336165613836613866
+31613838666530316635646133343230333465633935333066306235373431353431383035363233
+35626231383765363132343664373631356365353534376639653963386464396536313165656236
+36313933373263356334326633616262353339316563313864613539323530636462383932393330
+32326234666262623263353033306336346235343562303166626236336565656530666339383163
+34633635643535653064366137626562613730333566613434623931313931366562346534333435
+66623338646134343338313764633639346638383539393961656364643065663936663131616339
+65373664343264323039643936623337623430353433636465316461333233363664633863363036
+63636436303334313537353634633064336337326134623837396364383535326530613361626364
+37666666373236363364393735333834336638653162373965303534363738386535653966373730
+30386264356665623538383163613737313831616439643862616365363336356134663838376465
+39623030313936353865333933646366613461666365376334623162653865376432343533663663
+39623365656563356664646633626238356566316136623163303138323235323932623034616331
+33383631393430613165336535333736653938323962336432653638333138326139626566376132
+33363639613130343334666437343235633865396661303433623961643361623065653634366466
+38636566343035306638363161633532643839373534396432343462383565306166623630346432
+62323431643836353738363336346237386132373939303866623837396661363637343563626437
+64343365363134363334376565623366383531393832366562333937353665323135306137303736
+33623763343437336439663137613938316535386366333739633238323863373665623932346632
+34326463613135333336316465623661626461393938336533643038393566333637646462386665
+64393366656130306334366561663230643932353839306330613261303031373739386634346236
+62316230643230326436636164393264623730303163383933316463666231336664393631613435
+66623731373061336261343636623763313039333533663865626634306632656431663963383762
+32306265396561373333653237323534663038613931353565316330386537343262313937313932
+6631
diff --git a/hosts b/hosts
index 20f29fa..40d3d3c 100644
--- a/hosts
+++ b/hosts
@@ -1,9 +1,11 @@
[galaxyservers]
-usegalaxy.cz ansible_connection=local ansible_user=debian
+#usegalaxy.cz ansible_connection=local ansible_user=debian
+#galaxy-re.grid.cesnet.cz ansible_connection=local ansible_user=debian
+galaxy-umsa.grid.cesnet.cz ansible_connection=local ansible_user=debian
[dbservers:children]
galaxyservers
[monitoring]
-usegalaxy.cz ansible_connection=local ansible_user=debian
+galaxy-umsa.grid.cesnet.cz ansible_connection=local ansible_user=debian
[sentryservers]
-usegalaxy.cz ansible_connection=local ansible_user=debian
+galaxy-umsa.grid.cesnet.cz ansible_connection=local ansible_user=debian
diff --git a/roles/galaxyproject.nginx/tasks/ssl-common.yml b/roles/galaxyproject.nginx/tasks/ssl-common.yml
index 3e83bb5..1128aca 100644
--- a/roles/galaxyproject.nginx/tasks/ssl-common.yml
+++ b/roles/galaxyproject.nginx/tasks/ssl-common.yml
@@ -13,6 +13,17 @@
- reload nginx
- supervisorctl restart nginx
+#- name: Disable ssl_protocols in nginx.conf if set
+# lineinfile:
+# name: "{{ nginx_conf_file }}"
+# regexp: '^(\s*ssl_protocols\s.*)'
+# backrefs: yes
+# line: '#\1 # commented by Ansible'
+# backup: yes
+# notify:
+# - reload nginx
+# - supervisorctl restart nginx
+
- name: Configure server-wide SSL
template:
src: ssl.conf.j2
diff --git a/roles/galaxyproject.postgresql/defaults/main.yml b/roles/galaxyproject.postgresql/defaults/main.yml
index bd7cace..e8e6e46 100644
--- a/roles/galaxyproject.postgresql/defaults/main.yml
+++ b/roles/galaxyproject.postgresql/defaults/main.yml
@@ -33,7 +33,7 @@ __postgresql_pgdg_bin_dir: "{{ '/usr/pgsql-' ~ (postgresql_version | replace('.'
postgresql_backup_command: >-
{{ postgresql_backup_local_dir | quote }}/bin/backup.py
{{ '--rsync-connect-opts ' ~ (postgresql_backup_rsync_connect_opts | quote) if postgresql_backup_rsync_connect_opts else '' }}
- --rsync-backup-opts {{ postgresql_backup_rsync_backup_opts | quote }}
+ --rsync-backup-opts={{ postgresql_backup_rsync_backup_opts | quote }}
--keep {{ postgresql_backup_keep | quote }}
{{ '--pg-bin-dir ' ~ __postgresql_pgdg_bin_dir if ansible_os_family == 'RedHat' else '' }}
--backup --clean-archive {{ postgresql_backup_dir | quote }}
diff --git a/templates/galaxy/config/object_store_conf.xml.j2 b/templates/galaxy/config/object_store_conf.xml.j2
index 290d799..191a30f 100644
--- a/templates/galaxy/config/object_store_conf.xml.j2
+++ b/templates/galaxy/config/object_store_conf.xml.j2
@@ -2,12 +2,9 @@
-
-
-
-
-
-
+
+
+
diff --git a/templates/galaxy/config/oidc_backends_config.xml b/templates/galaxy/config/oidc_backends_config.xml
index 75b935d..23bebdb 100644
--- a/templates/galaxy/config/oidc_backends_config.xml
+++ b/templates/galaxy/config/oidc_backends_config.xml
@@ -1,12 +1,5 @@
-
- {{ e_infra_client_id }}
- {{ e_infra_client_secret }}
- https://{{ inventory_hostname }}/authnz/keycloak/callback
- https://login.e-infra.cz/oidc
- https://www.e-infra.cz/img/logo.svg
-
{{ elixir_client_id }}
{{ elixir_client_secret }}
diff --git a/templates/galaxy/config/sanitize_allowlist.txt b/templates/galaxy/config/sanitize_allowlist.txt
new file mode 100644
index 0000000..964e487
--- /dev/null
+++ b/templates/galaxy/config/sanitize_allowlist.txt
@@ -0,0 +1 @@
+toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc
diff --git a/templates/nginx/galaxy.j2 b/templates/nginx/galaxy.j2
index 9009161..1cbc9d6 100644
--- a/templates/nginx/galaxy.j2
+++ b/templates/nginx/galaxy.j2
@@ -15,7 +15,7 @@ server {
# Our log files will go to journalctl
access_log syslog:server=unix:/dev/log;
error_log syslog:server=unix:/dev/log;
- # client_body_temp_path /rbd/data/nginx_upload_store;
+ # client_body_temp_path {{ galaxy_data_prefix }}/nginx_upload_store;
client_body_buffer_size 1024m;
client_max_body_size 20G;
@@ -30,42 +30,6 @@ server {
proxy_set_header Upgrade $http_upgrade;
}
- location /_job_files {
- if ($request_method != POST) {
- rewrite "" /api/jobs/$arg_job_id/files last;
- }
- upload_store {{ galaxy_config['galaxy']['nginx_upload_job_files_store'] }};
- # the rate limit might be needed if we have many Pulsar jobs that are writing data back
- # we need to limit it because the disc IO can be too large, ideally this happens on a different node
- # upload_limit_rate 32k;
- upload_store_access user:rw group:rw all:rw;
- upload_pass_form_field "";
- upload_set_form_field "__${upload_field_name}_path" "$upload_tmp_path";
- upload_pass_args on;
- upload_pass /_upload_job_files_done;
- }
-
- location /_upload_job_files_done {
- internal;
- rewrite "" /api/jobs/$arg_job_id/files;
- }
-
- location /api/upload/resumable_upload {
- # Disable request and response buffering
- proxy_request_buffering off;
- proxy_buffering off;
- proxy_http_version 1.1;
-
- # Add X-Forwarded-* headers
- proxy_set_header X-Forwarded-Host $host;
- proxy_set_header X-Forwarded-Proto $scheme;
-
- proxy_set_header Upgrade $http_upgrade;
- proxy_set_header Connection "upgrade";
- client_max_body_size 0;
- proxy_pass http://localhost:{{ galaxy_tusd_port }}/files;
- }
-
# Static files can be more efficiently served by Nginx. Why send the
# request to Gunicorn which should be spending its time doing more useful
# things like serving Galaxy!
@@ -102,42 +66,24 @@ server {
alias /;
}
- # Support click-to-run in the GTN-in-Galaxy Webhook
- location /training-material/ {
- proxy_pass https://training.galaxyproject.org/training-material/;
- }
-
- location ~* /rabbitmq/(.*) {
- rewrite ^/rabbitmq/(.*)$ /$1 break;
- proxy_pass http://127.0.0.1:15672;
- proxy_buffering off;
- proxy_set_header Host $http_host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto $scheme;
- }
-
- location /flower {
- proxy_pass http://localhost:5555;
- proxy_set_header Host $host;
- proxy_redirect off;
- proxy_http_version 1.1;
- proxy_set_header Upgrade $http_upgrade;
- proxy_set_header Connection "upgrade";
- }
-
- location /reports/ {
- auth_basic galaxy;
- auth_basic_user_file /etc/nginx/passwd;
- proxy_pass http://{{ galaxy_config.gravity.reports.bind }}:/;
- }
-
- location /grafana/ {
- proxy_pass http://127.0.0.1:3000/;
- proxy_set_header Host $http_host;
- }
-
- {{ tiaas_nginx_routes }}
+# location ~* /rabbitmq/(.*) {
+# rewrite ^/rabbitmq/(.*)$ /$1 break;
+# proxy_pass http://127.0.0.1:15672;
+# proxy_buffering off;
+# proxy_set_header Host $http_host;
+# proxy_set_header X-Real-IP $remote_addr;
+# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+# proxy_set_header X-Forwarded-Proto $scheme;
+# }
+
+# location /flower {
+# proxy_pass http://localhost:5555;
+# proxy_set_header Host $host;
+# proxy_redirect off;
+# proxy_http_version 1.1;
+# proxy_set_header Upgrade $http_upgrade;
+# proxy_set_header Connection "upgrade";
+# }
}