diff --git a/dev_playbook.yml b/dev_playbook.yml index 07560399d..e596ad863 100644 --- a/dev_playbook.yml +++ b/dev_playbook.yml @@ -11,6 +11,7 @@ - secret_group_vars/stats_server_vault - secret_group_vars/dev_secrets - secret_group_vars/sentry_vault + - secret_group_vars/history_mailer_vault pre_tasks: - name: Attach volume to instance include_role: @@ -44,68 +45,69 @@ enabled: false state: stopped roles: - - galaxyproject.repos - - common - - mounts - - install-tpv - - geerlingguy.pip - - gantsign.golang - - cyverse-ansible.singularity - - galaxyproject.galaxy - - role: galaxyproject.miniconda - become: true - become_user: galaxy - - usegalaxy_eu.galaxy_subdomains - - webhooks - - nginx-upload-module - - galaxyproject.nginx - - galaxyproject.tusd - - geerlingguy.nfs - - galaxyproject.slurm - # - galaxyproject.s3fs - - galaxyproject.cvmfs - - galaxyproject.gxadmin - - pg-post-tasks - - remote-pulsar-cron - - galaxy-pg-cleanup - #- galaxyproject.tiaas2 # broken in galaxy release_23.1 - missing static/style/base.css - - geerlingguy.docker - - dj-wasabi.telegraf - - postfix-mail-relay - #- login-override - - acl-on-startup - post_tasks: - - name: Ensure object store paths exist - file: - state: directory - path: "{{ item }}" - owner: galaxy - group: galaxy - with_items: - - /mnt/galaxy/data - - /mnt/galaxy/data-2 - - /mnt/galaxy/data-3 - - name: Make local_tool directory group-writable by machine users - file: - path: /mnt/galaxy/local_tools - owner: root - group: devs - mode: 0775 - state: directory - - name: Install slurm-drmaa - package: - name: slurm-drmaa1 - - name: Uninstall the existing web-dav-client - pip: - name: "webdavclient3" - virtualenv: "{{ galaxy_venv_dir }}" - state: absent - - name: Workaround content-length header bug in webdav through forcible update to newer version - pip: - name: "webdavclient3@git+https://github.com/ezhov-evgeny/webdav-client-python-3@0f17fa7946e66f7963db367d0d6b2e7f940ebeb8" - virtualenv: "{{ galaxy_venv_dir }}" - - name: Reload exportfs - command: exportfs -ra - become: yes - become_user: root + # - galaxyproject.repos + # - common + # - mounts + # - install-tpv + # - geerlingguy.pip + # - gantsign.golang + # - cyverse-ansible.singularity + # - galaxyproject.galaxy + # - role: galaxyproject.miniconda + # become: true + # become_user: galaxy + # - usegalaxy_eu.galaxy_subdomains + # - webhooks + # - nginx-upload-module + # - galaxyproject.nginx + # - galaxyproject.tusd + # - geerlingguy.nfs + # - galaxyproject.slurm + # # - galaxyproject.s3fs + # - galaxyproject.cvmfs + # - galaxyproject.gxadmin + # - pg-post-tasks + # - remote-pulsar-cron + # - galaxy-pg-cleanup + # #- galaxyproject.tiaas2 # broken in galaxy release_23.1 - missing static/style/base.css + # - geerlingguy.docker + # - dj-wasabi.telegraf + # - postfix-mail-relay + # #- login-override + # - acl-on-startup + - usegalaxy-au.history_mailer + # post_tasks: + # - name: Ensure object store paths exist + # file: + # state: directory + # path: "{{ item }}" + # owner: galaxy + # group: galaxy + # with_items: + # - /mnt/galaxy/data + # - /mnt/galaxy/data-2 + # - /mnt/galaxy/data-3 + # - name: Make local_tool directory group-writable by machine users + # file: + # path: /mnt/galaxy/local_tools + # owner: root + # group: devs + # mode: 0775 + # state: directory + # - name: Install slurm-drmaa + # package: + # name: slurm-drmaa1 + # - name: Uninstall the existing web-dav-client + # pip: + # name: "webdavclient3" + # virtualenv: "{{ galaxy_venv_dir }}" + # state: absent + # - name: Workaround content-length header bug in webdav through forcible update to newer version + # pip: + # name: "webdavclient3@git+https://github.com/ezhov-evgeny/webdav-client-python-3@0f17fa7946e66f7963db367d0d6b2e7f940ebeb8" + # virtualenv: "{{ galaxy_venv_dir }}" + # - name: Reload exportfs + # command: exportfs -ra + # become: yes + # become_user: root diff --git a/galaxy-backup_playbook.yml b/galaxy-backup_playbook.yml index 5a335a58c..e4a0296d4 100644 --- a/galaxy-backup_playbook.yml +++ b/galaxy-backup_playbook.yml @@ -6,6 +6,7 @@ - group_vars/VAULT - host_vars/galaxy-backup.yml - secret_group_vars/stats_server_vault + - secret_group_vars/history_mailer_vault - secret_group_vars/ubuntu_maintenance_key pre_tasks: - name: Attach volume to instance @@ -18,6 +19,7 @@ - common - geerlingguy.pip - slg.db-backup + - usegalaxy-au.history_mailer - dj-wasabi.telegraf post_tasks: - name: Ensure desired version of ansible is installed diff --git a/galaxy-db_playbook.yml b/galaxy-db_playbook.yml index 7cf3c4663..0d6305ed2 100644 --- a/galaxy-db_playbook.yml +++ b/galaxy-db_playbook.yml @@ -7,6 +7,7 @@ - group_vars/dbservers.yml - host_vars/galaxy-db.usegalaxy.org.au.yml - secret_group_vars/stats_server_vault + - secret_group_vars/history_mailer_vault - secret_group_vars/ubuntu_maintenance_key pre_tasks: - name: Attach volume to instance diff --git a/group_vars/all.yml b/group_vars/all.yml index 42f04438a..09f15b829 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -158,6 +158,8 @@ common_packages: - fail2ban - software-properties-common - pigz + - sqlite3 + - iotop #Set pip to be pip3 by default - see roles/geerlingguy.pip/defaults/main.yml pip_package: python3-pip diff --git a/host_vars/dev.gvl.org.au.yml b/host_vars/dev.gvl.org.au.yml index 8e0c8e80d..732684d69 100644 --- a/host_vars/dev.gvl.org.au.yml +++ b/host_vars/dev.gvl.org.au.yml @@ -416,3 +416,61 @@ webhook_plugins: - demo - gtn - news + +### History mailer + +# config +history_mailer_warn_days: 365 +history_mailer_delete_days: 379 +history_mailer_email_days_threshold: 14 +history_mailer_purge_days_threshold: 6 + +history_mailer_galaxy_url: https://dev.gvl.org.au +history_mailer_galaxy_api_key: "{{ vault_jenkins_bot_dev_api_key }}" # TODO: add this! + +history_mailer_galaxy_keeplist_group: "History Retention Keeplist" + +# history_mailer_postal_base_url: "https://mail.usegalaxy.org.au/api/v1/" +# history_mailer_postal_api_key: "{{ vault_history_mailer_postal_api_key }}" + +# history_mailer_email_template_warning: "{{ history_mailer_email_template_dir }}/email_warning.html" +# history_mailer_email_template_deletion: "{{ history_mailer_email_template_dir }}/email_deletion.html" + +# history_mailer_mail_from: "Galaxy Australia " +# history_mailer_mail_replyto: "help@genome.edu.au" + +# history_mailer_slack_token: "{{ vault_history_mailer_slack_token }}" +# history_mailer_slack_alert_channel: "#alerts" +# history_mailer_slack_log_channel: "#galaxy-logs" +# history_mailer_alert_mentions: "<@Catherine>" +# history_mailer_log_mentions: "<@Catherine>" + +# settings for ansible role +history_mailer_user: ubuntu +history_mailer_dir: "/home/{{ history_mailer_user }}/hm_TEST" + +# history_mailer_use_postal: true # Must be true for history mailer to work + +history_mailer_enable_cron_jobs: false # When not enabled, cron jobs will be entered in crontab in a disabled state + +history_mailer_cron_jobs: +- name: warn_and_delete + weekday: "3" + hour: "11" + options: + - production + - warn + - dryrun +# - name: purge_histories +# weekday: "2" +# hour: "12" +# options: +# - production +# - purge +# - notify + +# Delete these and use vault variables +vault_history_mailer_postal_api_key: nottherealkey # xGV6VbNrHOGO8rirFL2e2E6b +vault_history_mailer_slack_token: "xoxb" +vault_jenkins_bot_production_api_key: abcdefg # TODO: add the real one, add these to vault +vault_jenkins_bot_staging_api_key: xyz # TODO: add the real one, add these to vault \ No newline at end of file diff --git a/host_vars/galaxy-backup.yml b/host_vars/galaxy-backup.yml index 303963718..3c3cdb96c 100644 --- a/host_vars/galaxy-backup.yml +++ b/host_vars/galaxy-backup.yml @@ -49,3 +49,70 @@ extra_keys: ssh_config_id_file: "/home/{{ ssh_config_user }}/.ssh/internal_hop_key" ssh_config_user: ubuntu ssh_config_hosts: "{{ groups['galaxy_group'] }}" + +### History mailer + +# config +history_mailer_warn_days: 365 +history_mailer_delete_days: 379 +history_mailer_email_days_threshold: 14 +history_mailer_purge_days_threshold: 6 + +history_mailer_galaxy_url: https://usegalaxy.org.au +history_mailer_galaxy_api_key: "{{ vault_jenkins_bot_production_api_key }}" # TODO: add this! +history_mailer_staging_galaxy_url: https://staging.gvl.org.au +history_mailer_staging_galaxy_api_key: "{{ vault_jenkins_bot_staging_api_key }}" # TODO: add this! + +history_mailer_galaxy_keeplist_group: "History Retention Keeplist" + +history_mailer_postal_base_url: "https://mail.usegalaxy.org.au/api/v1/" +history_mailer_postal_api_key: "{{ vault_history_mailer_postal_api_key }}" + +history_mailer_email_template_warning: "{{ history_mailer_email_template_dir }}/email_warning.html" +history_mailer_email_template_deletion: "{{ history_mailer_email_template_dir }}/email_deletion.html" + +history_mailer_mail_from: "Galaxy Australia " +history_mailer_mail_replyto: "help@genome.edu.au" + +history_mailer_slack_token: "{{ vault_history_mailer_slack_token }}" +history_mailer_slack_alert_channel: "#alerts" +history_mailer_slack_log_channel: "#galaxy-logs" +history_mailer_alert_mentions: "<@Catherine>" +history_mailer_log_mentions: "<@Catherine>" + +# settings for ansible role +history_mailer_user: ubuntu +history_mailer_dir: "/home/{{ history_mailer_user }}/hm_TEST" + +history_mailer_use_postal: true # Must be true for history mailer to work + +history_mailer_enable_cron_jobs: false # When not enabled, cron jobs will be entered in crontab in a disabled state + +history_mailer_files: + - src: templates/history_mailer/email_warning.html + dest: "{{ history_mailer_email_template_warning }}" + - src: templates/history_mailer/email_deletion.html + dest: "{{ history_mailer_email_template_deletion }}" + +history_mailer_cron_jobs: +- name: warn_and_delete + weekday: "3" + hour: "11" + options: + - production + - warn + - delete + - notify +- name: purge_histories + weekday: "2" + hour: "12" + options: + - production + - purge + - notify + +# Delete these and use vault variables +vault_history_mailer_postal_api_key: nottherealkey # +vault_history_mailer_slack_token: "xoxb" +vault_jenkins_bot_production_api_key: abcdefg # TODO: add the real one, add these to vault +vault_jenkins_bot_staging_api_key: xyz # TODO: add the real one, add these to vault diff --git a/requirements.yml b/requirements.yml index c6c1487f6..9fe219567 100644 --- a/requirements.yml +++ b/requirements.yml @@ -79,3 +79,7 @@ roles: - name: geerlingguy.redis version: 1.8.0 +- name: usegalaxy-au.history_mailer + src: https://github.com/usegalaxy-au/ansible-history-mailer + version: d6a5120 # TODO: at the very least add a commit ID here + diff --git a/scripts/update_roles.py b/scripts/update_roles.py index 9fce02138..6bc4e8861 100644 --- a/scripts/update_roles.py +++ b/scripts/update_roles.py @@ -30,7 +30,6 @@ role_requirements = yaml.safe_load(handle).get('roles') for r in role_requirements: - try: name = r.get('name', r.get('src')) if not name: raise Exception(f'Could not find role {name} in yaml entry') @@ -38,22 +37,24 @@ if not required_version: print(f'role {name} has no required version, no need to update') continue + role_info_file = os.path.join(here, roles_dir, name, 'meta', '.galaxy_install_info') - role_info_version = None - with open(role_info_file) as handle: - role_info_lines = handle.readlines() - for line in role_info_lines: - content = re.split(':\s+', line.strip()) - if content[0] == 'version': - role_info_version = content[1] - if role_info_version and role_info_version == required_version: - print(f'role {name} is already installed at version {role_info_version}, no need to update') - else: - print(f'role {name} will be updated from {role_info_version} to {required_version}') + if not os.path.exists(role_info_file): + print(f'role {name} has no .galaxy_install_info, assuming new and adding to roles to install') roles_to_update.append(r) - except Exception as e: # file doesnt exist or isn't parsed properly or something - sys.stderr.write(str(e)) - roles_to_update.append(r) + else: + role_info_version = None + with open(role_info_file) as handle: + role_info_lines = handle.readlines() + for line in role_info_lines: + content = re.split(':\s+', line.strip()) + if content[0] == 'version': + role_info_version = content[1] + if role_info_version and role_info_version == required_version: + print(f'role {name} is already installed at version {role_info_version}, no need to update') + else: + print(f'role {name} will be updated from {role_info_version} to {required_version}') + roles_to_update.append(r) if roles_to_update: with open(output_file, 'w') as handle: yaml.safe_dump(roles_to_update, handle) diff --git a/secret_group_vars/history_mailer_vault b/secret_group_vars/history_mailer_vault new file mode 100644 index 000000000..e3ce6291f --- /dev/null +++ b/secret_group_vars/history_mailer_vault @@ -0,0 +1,23 @@ +$ANSIBLE_VAULT;1.1;AES256 +31616565623561373966653435633261313061623030303666653565633133633066343434663433 +3364373839316366623839373838623236373365306230660a613235346463653064353765623030 +36316166326664643765623939343566633231333030646130343736366431666566386430363230 +6235643866386335640a633361303836663865343431616134363266623535623139616632353138 +61373737323061343261383862333736393761666534306439633034336564646261323363393238 +38626364376166663331386130333131656664633934323332316431376661656661303632393464 +38636238303935373265343837653731306635623964326639316436386463313163366334373039 +34626134353030623465303537653434333832303464663035326333653261393736393939326462 +33346532366335333434653539613861353838333539643035636333333464656437396133666161 +37366138386130613364333734323233323133393363663739663039326262343639626462313561 +37346162623963376561393633653230396337313763343763613530383630386437653237666665 +38643565383337613233616133643031393131623266646136663137353133643664653332373934 +66373764623861363738633331656436356534656262303433616436316637393635336665306166 +36633066393764616534303035613236386332633739626466623439353933383965316665366465 +30623432383839396465646434313536333066663064323066343334386566323234353438336632 +32643538363239373734623166333466653265303937393736363837666266363832363366313862 +34643764303736343732666536343465636230393636303739333338303164653138383238343263 +63663633373830626461336639626663356435343531626561336164663136666134613361326634 +31303937623063313663616531626530353065356138646336323131356430343933646434373439 +66663661623465326139326139393131356238306631383432303037623035333862323763653130 +64663230353835646436323538383662633539643866363063643732313132343537353665396462 +6566303835656131353431666435356362366633316338626466 diff --git a/templates/history_mailer/email_deletion.html b/templates/history_mailer/email_deletion.html new file mode 100644 index 000000000..36894d562 --- /dev/null +++ b/templates/history_mailer/email_deletion.html @@ -0,0 +1,24 @@ + + +

Dear {{ username }},

+

You are receiving this email as one or more of your histories on Galaxy Australia have not been updated for {{ delete_weeks }} weeks or more and have now been marked as deleted. They and their associated data will be purged from our disk in 5 days time (from the date of this email). + +

Please see the {% if histories|length > 1 %}histories{% else %}history{% endif %} in question below:

+

+ + + {% for h in histories %} + + {% endfor %} +
History NameDate last updatedSize
{{ h['name'] }}{{ h['h_update_time'] }}{{ h['h_size'] }}
+

+

If you have any queries regarding this email, please don't hesitate to reply to: help@genome.edu.au

+ Yours, +
+
+ Galaxy Australia Administrators. +

+ + + diff --git a/templates/history_mailer/email_warning.html b/templates/history_mailer/email_warning.html new file mode 100644 index 000000000..002c24810 --- /dev/null +++ b/templates/history_mailer/email_warning.html @@ -0,0 +1,34 @@ + + +

Dear {{ username }},

+

You are receiving this email as one or more of your histories on Galaxy Australia have not been updated for {{ warn_weeks }} weeks or more. They will be beyond the User Data Storage time limits soon ({{ delete_weeks }} weeks). Displayed next to each history in the table below is the date that it will be deleted. If you do not run a job in that history or update it before that date, it will be automatically deleted and then purged from disk.

+ +

You should download any files you wish to keep from each history before the date specified. Instructions for doing so can be found at:

+ +

Galaxy Training Material - Downloading and Deleting Data

+ +

Please note that if you have received a warning email in the past, new histories could have been added to the following table and they will have a different scheduled deletion date.

+ +

Please see the {% if histories|length > 1 %}histories{% else %}history{% endif %} in question below:

+

+ + + {% for h in histories | sort(attribute='h_update_time') %} + + {% endfor %} +
History NameDate last updatedSizeDeletion Date
{{ h['name'] }}{{ h['h_update_time'] }}{{ h['h_size'] }}{{ h['h_del_time'] }}
+

+ + + +

Galaxy Australia is a data analysis platform and stores data in accordance with the User Data Storage Policy

+

If you have any queries regarding this email, please don't hesitate to contact us at: help@genome.edu.au

+ Yours, +
+
+ Galaxy Australia Administrators. +

+ + +