Merge branch 'main' of https://github.com/UniversityRadioYork/ury-bac…

…kups into main

README.md

@@ -1,3 +1,35 @@
-yeah, this can get written
+# URY's Server Backups
+
+## `urysteve` (primary filestore)
+- `backup_dbs.sh` - run daily, takes SQL dumps of our databses, and rsyncs them to `stratford` and `moyles`
+- `backups-hourly.sh` - runs hourly, sends all the `filestore` and `musicstore` snapshots to `stratford` and `moyles`
+- `sanoid` runs to snapshot the `filestore` and `musicstore`
+
+## `urybackup0` (on-site backup server)
+## RIP
+- `sanoid` runs to prune old `filestore`/`musicstore` snapshots, and to snapshot `pool0`
+- `rsync-daily.new.sh` runs to backup all the other servers we have, i.e. home directories. these are backed up to `pool0`
+
+## `stratford` (on-site backup server)
+- `sanoid` runs to prune old `filestore`/`musicstore` snapshots, and to snapshot `backup/servers`
+TODO:
+    - `rsync-daily.new.sh`
+    - `backup-alerts.sh`
+
+## `moyles` (offsite backup server)
+- `sanoid` runs to prune old `filestore`/`musicstore`/`pool0` snapshots
+TODO - add `rsync-daily.new.sh` to get moyles backups to be independent of on site backups, and also change the sanid config as needed (i.e. copy stratford)
+
+**Both `urybackup0` and `moyles` run `backup-alerts.sh` daily to send any important information to us in Slack, such as:**
+- missing snapshots
+- ZFS pools not being OK
+- running out of storage space
+
+
+## `uryrrod` (studio webcam recording server)
+- `movempg.sh` transfers recorded footage to the `filestore`, where `urysteve` will also tidy up older recordings (older than 90 days)
+
+## TODO
+- mailbox backups - other than just the `ury` server backup
+- external monitoring of `backup-alerts.sh` script actually running
 
-and we desperately need some monitoring scripts

backup-alerts.sh

@@ -9,31 +9,30 @@ set -eo pipefail
 
 set -u
 
-# we love how `date` is different on BSD and Linux :o
 BACKUP_SERVER=$(hostname)
 case $BACKUP_SERVER in
-    "urybackup0")
-        musicstore="music"
-        filestore="pool1"
-        database="pool0/backup/db"
-        server_backup="pool0/backup"
-        yesterday=$(date -r $(($(date +'%s') - 86400)) +'%Y-%m-%d')
+    "stratford")
+        musicstore="musicstore"
+        filestore="filestore"
+        database="backup/db"
+        server_backup="backup/servers"
         ;;
 
     "moyles")
         musicstore="pool0/music"
         filestore="pool0/pool1"
         database="pool0/db"
         server_backup="pool0/backup"
-        yesterday=$(date -d yesterday +'%Y-%m-%d')
         ;;
 
     *)
-        echo >&2 "this isn't running on bup0 or moyles"
+        echo >&2 "this isn't running on stratford or moyles"
         exit 1
         ;;
 esac
 
+yesterday=$(date -d yesterday +'%Y-%m-%d')
+
 tmp_file=/tmp/$(date +'%s').backupalert
 alerts_started=0
 
@@ -50,6 +49,17 @@ daily_snapshot_exist () {
     zfs list -t snapshot | grep "${dataset}@autosnap_${date}_.*_daily"
 }
 
+weekly_running_check () {
+    if [[ $(date +'%w') != 3 ]]; then
+        return 0;
+    fi
+
+    if [[ alerts_started -eq 1 ]]; then
+        return 0;
+    fi
+
+    alert "No problems, just to say, I'm still checking for you :)"
+}
 
 for dataset in $musicstore $filestore $database $server_backup; do
         [[ $(daily_snapshot_exist $dataset $yesterday) == "" ]] && alert "$dataset missing daily snapshot"
@@ -69,6 +79,7 @@ check_server="urysteve/root"
 [[ $(find "/$server_backup/$check_server" -mtime -1 | head -1) == "" ]] && alert "No backup data on $server_backup (checked $check_server)"
 [[ $(find "/$database" -mtime -1 | head -1) == "" ]] && alert "No new database files"
 
+weekly_running_check
 [[ -f $tmp_file ]] && curl -X POST --data-urlencode "payload={\"text\": \"$(cat $tmp_file)\"}" $BACKUP_ALERT_SLACK_HOOK
 
-rm $tmp_file 2>/dev/null    
+rm $tmp_file 2>/dev/null    

moyles/sanoid.conf

@@ -1,7 +1,6 @@
 [pool0/backup]
         use_template = production
         recursive = yes
-        autosnap = no
         hourly = 0
 [pool0/pool1]
         use_template = production

urybackup0/rsync/rsync-daily.new.sh → rsync/rsync-daily.new.sh

@@ -15,9 +15,24 @@ log () {
         echo -e $(date) '\t' $1 '\t' ${@:2}
 }
 
-BACKUP_ROOT=/pool0/backup
-EXCLUDE_FILE=$BACKUP_ROOT/urybackup-exclude.new.conf
-SERVER_LIST=$BACKUP_ROOT/servers
+case $(hostname) in
+        "stratford")
+                BACKUP_ROOT="/backup/servers"
+                ;;
+
+        "moyles")
+                BACKUP_ROOT="/pool0/backup"
+                ;;
+
+        *)
+                echo >&2 "this isn't running on stratford or moyles"
+                exit 1
+                ;;
+esac
+
+
+EXCLUDE_FILE=/opt/ury-backups/rsync/urybackup-exclude.new.conf
+SERVER_LIST=/opt/ury-backups/rsync/servers
 
 log root Backups Started
 while read server; do
@@ -31,7 +46,7 @@ while read server; do
         else
                 log $server_name established connection - starting rsync
                 # Rsync Command
-                (rsync -ah --stats --delete --delete-delay --delete-excluded --exclude-from=$EXCLUDE_FILE --exclude-from=$BACKUP_ROOT/exclusions/$server_name $server:/ $BACKUP_ROOT/$server_name && log $server_name rsync complete) &
+                (rsync -ah --stats --delete --delete-delay --delete-excluded --exclude-from=$EXCLUDE_FILE --exclude-from=/opt/ury-backups/rsync/exclusions/$server_name $server:/ $BACKUP_ROOT/$server_name && log $server_name rsync complete) &
         fi
 done < $SERVER_LIST
 wait

urybackup0/rsync/servers → rsync/servers

@@ -4,4 +4,5 @@ dolby.ury.york.ac.uk
 ury.york.ac.uk
 urybsod.york.ac.uk
 uryrrod.york.ac.uk
-#uryfw0.york.ac.uk
+uryfw0.york.ac.uk
+evergiven.ury.york.ac.uk

stratford/sanoid.conf

@@ -0,0 +1,25 @@
+[backup]
+        autosnap = no
+[backup/servers]
+        use_template = production
+        recursive = yes
+        hourly = 0
+[backup/db]
+        use_template = production
+        hourly = 0
+[filestore]
+        use_template = production
+        recursive = yes
+        autosnap = no
+[musicstore]
+        use_template = production
+        autosnap = no
+
+[template_production]
+        frequently = 0
+        hourly = 24
+        daily = 30
+        monthly = 6
+        yearly = 0
+        autosnap = yes
+        autoprune = yes

uryrrod/cron

@@ -0,0 +1,2 @@
+# Move the webcam footage over to the filestore
+0       */6     *       *       *       root    /home/motion/movempg.sh

uryrrod/movempg.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+cd /home/motion
+rsync -a --remove-sent-files --files-from=<(find . -mtime +8m -name "*.mpg") . urysteve.ury:/filestore/Webcams

urysteve/backup_dbs.sh

@@ -12,16 +12,20 @@ for db in ${PG_DATABASES[@]}; do
  (export PGPASSWORD="$PG_PASSWORD" && /usr/bin/pg_dump -U "$PG_USERNAME" -F custom -f "$LOCAL_BACKUP_PATH/$db.sql.dump" "$db")
 done
 
-for server in "urybackup0.york.ac.uk" "moyles.ury.york.ac.uk"; do
+for server in "stratford.ury.york.ac.uk" "moyles.ury.york.ac.uk"; do
  # i hate this
  if [[ $server == "moyles.ury.york.ac.uk" ]]; then
         remote_path=/pool0/db
  else
-        remote_path=/pool0/backup/db
+        remote_path=/backup/db
  fi
-
- ping -c 1 $server > /dev/null 2>&1
- if [[ $? -eq 0 ]]; then
+
+ set +e
+ nc -vz $server 22 > /dev/null 2>&1
+ ping_ok=$?
+ set -e
+
+ if [[ $ping_ok -eq 0 ]]; then
         echo $(date): transferring to $server
         rsync -ah --stats --delete --delete-delay --delete-excluded $LOCAL_BACKUP_PATH/* $server:$remote_path
         echo $(date): transfer to $server done
@@ -30,4 +34,4 @@ for server in "urybackup0.york.ac.uk" "moyles.ury.york.ac.uk"; do
  fi
 
 done
-echo "$(date): done"
+echo "$(date): done"

urysteve/backups-hourly.sh

@@ -1,24 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 LOG=/var/log/replication.log
 
-onsite () {
- echo "`date` - syncing music-backup0" | tee -a $LOG
- /usr/sbin/syncoid -r --no-sync-snap --sshkey="/usr/local/share/backup/steve-bup0-rsa" music [email protected]:music | tee -a $LOG 2>&1
- echo "`date` - syncing filestore-backup0..." | tee -a $LOG
- /usr/sbin/syncoid -r --no-sync-snap --sshkey="/usr/local/share/backup/steve-bup0-rsa" filestore [email protected]:pool1 | tee -a $LOG 2>&1
-}
+/usr/sbin/syncoid -r --no-sync-snap music [email protected]:musicstore | tee -a $LOG 2>&1
+/usr/sbin/syncoid -r --no-sync-snap music [email protected]:pool0/music | tee -a $LOG 2>&1
+/usr/sbin/syncoid -r filestore [email protected]:filestore | tee -a $LOG 2>&1
+/usr/sbin/syncoid -r filestore [email protected]:pool0/pool1 | tee -a $LOG 2>&1
 
-offsite () {
- echo "`date` - syncing filestore-moyles..." | tee -a $LOG
- /usr/sbin/syncoid -r --no-sync-snap --sshkey="/usr/local/share/backup/steve-moyles-rsa" filestore [email protected]:pool0/pool1 | tee -a $LOG 2>&1
- echo "`date` - syncing music-moyles" | tee -a $LOG
- /usr/sbin/syncoid -r --no-sync-snap --sshkey="/usr/local/share/backup/steve-moyles-rsa" music [email protected]:pool0/music | tee -a $LOG 2>&1
-}
-
-
-
-onsite &
-offsite &
-wait
 echo "`date` - replication complete." | tee -a $LOG

urysteve/cron

@@ -5,4 +5,7 @@
 0 23 * * * root /usr/local/bin/backup_dbs.sh
 
 # this is from root's crontab
-10 * * * * /usr/local/sbin/backups-hourly.sh
+10 * * * * /usr/local/sbin/backups-hourly.sh
+
+# Prune Webcam Footage
+20      4       *       *       *       root    find /filestore/Webcams -mtime +90 -delete

urysteve/sanoid.conf

@@ -3,7 +3,7 @@
  recursive = yes
 [filestore/Webcams]
  autosnap = no
-[filestore/Archive]
+[filestore/Shows/AutoViz]
  autosnap = no
 [music]
  use_template = production
@@ -15,4 +15,4 @@ daily = 30
 monthly = 6
 yearly = 0
 autosnap = yes
-autoprune = yes
+autoprune = yes