Use history mailer ansible role

galaxy-backup_playbook.yml

@@ -6,6 +6,7 @@
       - group_vars/VAULT
       - host_vars/galaxy-backup.yml
       - secret_group_vars/stats_server_vault
+      - secret_group_vars/history_mailer_vault
       - secret_group_vars/ubuntu_maintenance_key
   pre_tasks:
     - name: Attach volume to instance
@@ -18,6 +19,7 @@
     - common
     - geerlingguy.pip
     - slg.db-backup
+    - usegalaxy-au.history_mailer
     - dj-wasabi.telegraf
   post_tasks:
     - name: Ensure desired version of ansible is installed

galaxy-db_playbook.yml

@@ -7,6 +7,7 @@
       - group_vars/dbservers.yml
       - host_vars/galaxy-db.usegalaxy.org.au.yml
       - secret_group_vars/stats_server_vault
+      - secret_group_vars/history_mailer_vault
       - secret_group_vars/ubuntu_maintenance_key
   pre_tasks:
       - name: Attach volume to instance

group_vars/all.yml

@@ -158,6 +158,8 @@ common_packages:
     - fail2ban
     - software-properties-common
     - pigz
+    - sqlite3
+    - iotop
 
 #Set pip to be pip3 by default - see roles/geerlingguy.pip/defaults/main.yml
 pip_package: python3-pip

host_vars/galaxy-backup.yml

@@ -49,3 +49,70 @@ extra_keys:
 ssh_config_id_file: "/home/{{ ssh_config_user }}/.ssh/internal_hop_key"
 ssh_config_user: ubuntu
 ssh_config_hosts: "{{ groups['galaxy_group'] }}"
+
+### History mailer
+
+# config
+history_mailer_warn_days: 365
+history_mailer_delete_days: 379
+history_mailer_email_days_threshold: 14
+history_mailer_purge_days_threshold: 6
+
+history_mailer_galaxy_url: https://usegalaxy.org.au
+history_mailer_galaxy_api_key: "{{ vault_jenkins_bot_production_api_key }}"  # TODO: add this!
+history_mailer_staging_galaxy_url: https://staging.gvl.org.au
+history_mailer_staging_galaxy_api_key: "{{ vault_jenkins_bot_staging_api_key }}"  # TODO: add this!
+
+history_mailer_galaxy_keeplist_group: "History Retention Keeplist"
+
+history_mailer_postal_base_url: "https://mail.usegalaxy.org.au/api/v1/"
+history_mailer_postal_api_key: "{{ vault_history_mailer_postal_api_key }}"
+
+history_mailer_email_template_warning: "{{ history_mailer_email_template_dir }}/email_warning.html"
+history_mailer_email_template_deletion: "{{ history_mailer_email_template_dir }}/email_deletion.html"
+
+history_mailer_mail_from: "Galaxy Australia <[email protected]>"
+history_mailer_mail_replyto: "[email protected]"
+
+history_mailer_slack_token: "{{ vault_history_mailer_slack_token }}"
+history_mailer_slack_alert_channel: "#alerts"
+history_mailer_slack_log_channel: "#galaxy-logs"
+history_mailer_alert_mentions: "<@Catherine>"
+history_mailer_log_mentions: "<@Catherine>"
+
+# settings for ansible role
+history_mailer_user: ubuntu
+history_mailer_dir: "/home/{{ history_mailer_user }}/hm_TEST"
+
+history_mailer_use_postal: true # Must be true for history mailer to work
+
+history_mailer_enable_cron_jobs: false # When not enabled, cron jobs will be entered in crontab in a disabled state
+
+history_mailer_files:
+  - src: templates/history_mailer/email_warning.html
+    dest: "{{ history_mailer_email_template_warning }}"
+  - src: templates/history_mailer/email_deletion.html
+    dest: "{{ history_mailer_email_template_deletion }}"
+
+history_mailer_cron_jobs:
+- name: warn_and_delete
+  weekday: "3"
+  hour: "11"
+  options:
+    - production
+    - warn
+    - delete
+    - notify
+- name: purge_histories
+  weekday: "2"
+  hour: "12"
+  options:
+    - production
+    - purge
+    - notify
+
+# Delete these and use vault variables
+vault_history_mailer_postal_api_key: nottherealkey # xGV6VbNrHOGO8rirFL2e2E6b
+vault_history_mailer_slack_token: "xoxb"
+vault_jenkins_bot_production_api_key: abcdefg # TODO: add the real one, add these to vault
+vault_jenkins_bot_staging_api_key: xyz # TODO: add the real one, add these to vault

requirements.yml

@@ -79,3 +79,7 @@ roles:
 - name: geerlingguy.redis
   version: 1.8.0
 
+- name: usegalaxy-au.history_mailer
+  src: https://github.com/usegalaxy-au/ansible-history-mailer
+  version: e01d8f1 # TODO: at the very least add a commit ID here
+

scripts/update_roles.py

@@ -30,30 +30,31 @@
     role_requirements = yaml.safe_load(handle).get('roles')
 
 for r in role_requirements:
-    try:
         name = r.get('name', r.get('src'))
         if not name:
             raise Exception(f'Could not find role {name} in yaml entry')
         required_version = r.get('version')
         if not required_version:
             print(f'role {name} has no required version, no need to update')
             continue
+
         role_info_file = os.path.join(here, roles_dir, name, 'meta', '.galaxy_install_info')
-        role_info_version = None
-        with open(role_info_file) as handle:
-            role_info_lines = handle.readlines()
-        for line in role_info_lines:
-            content = re.split(':\s+', line.strip())
-            if content[0] == 'version':
-                role_info_version = content[1]
-        if role_info_version and role_info_version == required_version:
-            print(f'role {name} is already installed at version {role_info_version}, no need to update')
-        else:
-            print(f'role {name} will be updated from {role_info_version} to {required_version}')
+        if not os.path.exists(role_info_file):
+            print(f'role {name} has no .galaxy_install_info, assuming new and adding to roles to install')
             roles_to_update.append(r)
-    except Exception as e: # file doesnt exist or isn't parsed properly or something
-        sys.stderr.write(str(e))
-        roles_to_update.append(r)
+        else:
+            role_info_version = None
+            with open(role_info_file) as handle:
+                role_info_lines = handle.readlines()
+            for line in role_info_lines:
+                content = re.split(':\s+', line.strip())
+                if content[0] == 'version':
+                    role_info_version = content[1]
+            if role_info_version and role_info_version == required_version:
+                print(f'role {name} is already installed at version {role_info_version}, no need to update')
+            else:
+                print(f'role {name} will be updated from {role_info_version} to {required_version}')
+                roles_to_update.append(r)
 if roles_to_update:
     with open(output_file, 'w') as handle:
         yaml.safe_dump(roles_to_update, handle)

secret_group_vars/history_mailer_vault

@@ -0,0 +1,20 @@
+$ANSIBLE_VAULT;1.1;AES256
+65396138306261343536613661613663363030613266373439363961646666636630326339303832
+3464323933373361646266373134613331393138306165330a393736326631636630363234313536
+64356237666163386261356364313634306266333733386464666132386131633862623836393862
+6166373261643361660a643566373634666536373330303265393531343263373466623938666664
+62626166393361653061346334363933386164623863623031393033343538313966656638393136
+62353238353361343638633364363036316438383336383535353261636238623033653531346432
+39353763363366613933643563633263363830393835326164373864656534363639303939353666
+38336230326565353261333764336633346463343032303562373236376330393732333636653333
+65636162396663613031396138313036626337626237343937643265353630393434363439316131
+39656139343039373061613737326537383463373533376234376437393862396565386137363165
+39613833616463626534386264376139343066653331646236363965663531636430376330613936
+66623734393339646635616461316139643965666365616266346532366137306536393536353130
+65643734346439383565643437656664623838663964623835303239333962366236373663633962
+31386465353861646339663137353139643037313562623964616166366566323834346666626535
+64613530313463373635623437663930663139613839316430313132333363306238666238663762
+35393730633365393965643636643435653666326164373533386264363733353833663861316237
+33386638613434306135353966393537663065633337373139633962663662633862376435383730
+65656165353764326434373235633837613332373435663830623639653862626636323363356432
+353164613861343265396138313562393366