starting modifications for galaxy-umsa

ansible.cfg

@@ -1,14 +1,16 @@
 [defaults]
 interpreter_python = /usr/bin/python3
 inventory = hosts
+bin_ansible_callbacks = True
 retry_files_enabled = false
 vault_password_file = .vault-password.txt
 gather_subset=!hardware
-no_log = True
+#no_log = True
+log_path = /var/log/ansible.log
 
 # Show diffs of changes
 [diff]
-always = False
+always = True
 
 [ssh_connection]
-pipelining = true
+pipelining = True

galaxy.yml

@@ -5,20 +5,58 @@
   pre_tasks:
     - name: Install Dependencies
       package:
-        name: 'acl'
+        name: ['acl', 'logrotate']
   roles:
     - galaxyproject.postgresql
     - role: galaxyproject.postgresql_objects
       become: true
       become_user: postgres
+  post_tasks:
+    - name: Create pg_dump backup folder if needed
+      file:
+        path:  "{{ postgresql_backup_dir }}/pg_dump"
+        state: directory
+        owner: postgres
+        group: "{{ galaxy_user_group }}"
+        mode:  0750
+    - name: Create initial empty pg_dump file if needed # otherwise the following logrotate conf will not work
+      file:
+        path:  "{{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql.gz"
+        state: touch
+        owner: postgres
+        group: "{{ galaxy_user_group }}"
+        mode:  0640
+    - name: Set-up Galaxy DB backup using pg_dump and logrotate
+      ansible.builtin.blockinfile:
+        path: "/etc/logrotate.d/{{ postgresql_pg_dump_prefix }}"
+        create: true
+        block: |
+          {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql.gz {
+            daily
+            missingok
+            rotate 14
+            nocompress
+            nocreate
+            postrotate
+              echo "### Start backup at `date` ###" >> /var/log/{{ postgresql_pg_dump_prefix }}_errors.log
+              sudo -u postgres pg_dump {{ galaxy_db_name }} --file={{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql --verbose >> /var/log/{{ postgresql_pg_dump_prefix }}_errors.log 2>&1
+              sudo chmod 640 {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql >> /var/log/{{ postgresql_pg_dump_prefix }}_errors.log 2>&1
+              sudo -u postgres gzip {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql >> /var/log/{{ postgresql_pg_dump_prefix }}_errors.log 2>&1
+              echo "### Finished backup at `date` ###" >> /var/log/{{ postgresql_pg_dump_prefix }}_errors.log
+            endscript
+          }
 
 - hosts: galaxyservers
   become: true
   become_user: root
   vars_files:
     - group_vars/secret.yml
   pre_tasks:
-    - pip:
+    - name: Install Dependencies
+      ansible.builtin.package:
+        name: ['acl', 'bzip2', 'git', 'make', 'tar', 'python3-bioblend', 'python3-venv', 'python3-setuptools', 'python3-pip', 'python3-psycopg2', 'rsync', 'python3-docker', 'nginx-full']
+    - name: Install additional pip packages
+      ansible.builtin.pip:
         name: passlib
     - htpasswd:
         path: /etc/nginx/passwd
@@ -27,9 +65,6 @@
         owner: www-data    # nginx on centos
         group: root
         mode: 0640
-    - name: Install Dependencies
-      package:
-        name: ['acl', 'bzip2', 'git', 'make', 'tar', 'python3-bioblend', 'python3-venv', 'python3-setuptools']
     - name: Install RHEL/CentOS/Rocky specific dependencies
       package:
         name: ['tmpwatch']
@@ -38,30 +73,57 @@
       package:
         name: ['tmpreaper']
       when: ansible_os_family == 'Debian'
+    - name: Ensure the python command exists
+      ansible.builtin.shell: 
+        cmd: which python||exit 0
+      register: python_path
+    - name: Register python3 path
+      ansible.builtin.command: which python3
+      register: python3_path
+      when: python_path.stdout == ""
+    - name: Link python to python3 if missing
+      ansible.builtin.file:
+        src: python3_path.stdout
+        dest: python3_path.stdout[:-1]
+        state: link
+        force: yes
+      when: python_path.stdout == ""
     - git:
         repo: 'https://github.com/usegalaxy-eu/libraries-training-repo'
         dest: /libraries/
+      when: "'usegalaxy' in inventory_hostname"
+#    - name: Create data storage for UMSA
+#      file:
+#        owner: "{{ galaxy_user_name }}"
+#        group: "{{ galaxy_user_group }}"
+#        path:  "{{ galaxy_data_prefix }}"
+#        state: directory
+#        mode:  "0755"
+#      when: "'umsa' in inventory_hostname"
     - name: Create the RBD storage directory
       file:
-        owner: galaxy
-        group: galaxy
+        owner: "{{ galaxy_user_name }}"
+        group: "{{ galaxy_user_group }}"
         path: /rbd/data
         state: directory
         mode: '0755'
+      when: "'usegalaxy' in inventory_hostname"
     - name: Create the nginx upload job files store
       file:
         owner: www-data
-        group: galaxy
+        group: "{{ galaxy_user_group }}"
         path: /rbd/data/_upload_job_files
         state: directory
         mode: '0775'
+      when: "'usegalaxy' in inventory_hostname"
     - name: Create the client body temp path   
       file:
         owner: www-data
-        group: galaxy
+        group: "{{ galaxy_user_group }}"
         path: /rbd/data/nginx_upload_store
         state: directory
         mode: '0775'
+      when: "'usegalaxy' in inventory_hostname"
   roles:
     - galaxyproject.tusd
     - usegalaxy_eu.apptainer
@@ -76,15 +138,15 @@
     - galaxyproject.proftpd
     - geerlingguy.docker
     - usegalaxy_eu.rabbitmqserver
-    - galaxyproject.tiaas2
+#    - galaxyproject.tiaas2
     - galaxyproject.gxadmin
     - galaxyproject.cvmfs
-    - dj-wasabi.telegraf
+#    - dj-wasabi.telegraf
   post_tasks:
     - name: Setup gxadmin cleanup task
       ansible.builtin.cron:
         name: "Cleanup Old User Data"
-        user: galaxy # Run as the Galaxy user
+        user: "{{ galaxy_user_name }}" # Run as the Galaxy user
         minute: "0"
         hour: "0"
         job: "GALAXY_LOG_DIR=/tmp/gxadmin/ GALAXY_ROOT={{ galaxy_root }}/server /usr/local/bin/gxadmin galaxy cleanup 60"
@@ -95,3 +157,11 @@
         minute: "0"
         hour: "*"
         job: "python3 {{ playbook_dir }}/add_group_users.py {{ inventory_hostname }} {{ api_key }}"
+      when: "'usegalaxy' in inventory_hostname"
+    - name: RE Galaxy instance patch in lib/galaxy/webapps/galaxy/api/job_files.py
+      replace:
+        path: "{{ galaxy_root }}/server/lib/galaxy/webapps/galaxy/api/job_files.py"
+        regexp: '^(\s*?)shutil\.move\(input_file\.name,\s*path\)\s*$'
+        replace: '\1os.chmod(shutil.move(input_file.name, path), 0o664)'
+        validate: 'python -m py_compile %s'
+      when: "'galaxy-re' in inventory_hostname or 'galaxy-re' in ansible_hostname"

group_vars/all.yml

@@ -3,8 +3,11 @@
 pip_virtualenv_command: /usr/bin/python3 -m venv  # usegalaxy_eu.certbot, usegalaxy_eu.tiaas2, galaxyproject.galaxy
 
 # Common variables needed by all hosts
-galaxy_user_name: galaxy
+galaxy_user_name: galaxyumsa
+galaxy_user_id: 10456
+galaxy_user_group: galaxyumsa
 galaxy_db_name: galaxy
+galaxy_data_prefix: "/mnt/data"
 
 # CVMFS vars
 cvmfs_role: client

group_vars/dbservers.yml

@@ -13,28 +13,28 @@ postgresql_objects_privileges:
     privs: SELECT
     objs: ALL_IN_SCHEMA
 
-  - database: galaxy
-    roles: tiaas
-    objs: galaxy_user,galaxy_session,job,history,workflow,workflow_invocation
-    type: table
-    privs: SELECT
-  - database: galaxy
-    roles: tiaas
-    objs: user_group_association,galaxy_group,role,group_role_association
-    type: table
-    privs: SELECT,INSERT
-  - database: galaxy
-    roles: tiaas
-    objs: group_role_association
-    type: table
-    privs: DELETE
-  - database: galaxy
-    roles: tiaas
-    objs: role_id_seq,galaxy_group_id_seq,group_role_association_id_seq,user_group_association_id_seq
-    type: sequence
-    privs: USAGE,SELECT
+#  - database: galaxy
+#    roles: tiaas
+#    objs: galaxy_user,galaxy_session,job,history,workflow,workflow_invocation
+#    type: table
+#    privs: SELECT
+#  - database: galaxy
+#    roles: tiaas
+#    objs: user_group_association,galaxy_group,role,group_role_association
+#    type: table
+#    privs: SELECT,INSERT
+#  - database: galaxy
+#    roles: tiaas
+#    objs: group_role_association
+#    type: table
+#    privs: DELETE
+#  - database: galaxy
+#    roles: tiaas
+#    objs: role_id_seq,galaxy_group_id_seq,group_role_association_id_seq,user_group_association_id_seq
+#    type: sequence
+#    privs: USAGE,SELECT
 
 # PostgreSQL Backups
-postgresql_backup_dir: /data/backups
-postgresql_backup_local_dir: "{{ '~postgres' | expanduser }}/backups"
-
+postgresql_backup_dir: "{{ galaxy_data_prefix }}/backups"
+postgresql_backup_local_dir: "{{ galaxy_data_prefix }}/backups/scripts"
+postgresql_pg_dump_prefix: galaxy_db_backup