-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathbackup_wiki.sh
executable file
·172 lines (154 loc) · 5.46 KB
/
backup_wiki.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
#!/bin/bash
##-- setup
PATH=/ccb/sw/bin:~/bin:/usr/bin:/bin:/sbin:/usr/sbin
LD_LIBRARY_PATH=/ccb/sw/lib
bn=bck_$(date +%y%m%d_%H-%M)
host=$(hostname -s)
## use ~/.my.cnf instead:
#dbuser=$USER
#read -p 'User:' dbuser
#dbpass=""
#read -s -p 'Password:' dbpass
##!IMPORTANT: do not place '/' at the end of these paths!
#wikidir=/home/html/genomics/igmwiki
wikidir=/ccb/salz7-data/www/ccb.jhu.edu/html/wiki
#rdirs=( "geom.us.to:/mylocal/geo/backups/igmwiki" "igm3.pha.jhu.edu:/home/gpertea/backups/wiki" )
rdirs=( "home:/mylocal/geo/backups/ccbwiki" "maestro:/data/backups/ccbwiki" "gwork:/mylocal/geo/backups/ccbwiki")
ferr=~/backups/wiki/$bn.stderr
cd $wikidir
##- check if LocalSettings.php has changed
files_changed=''
if ! cmp -s ./LocalSettings.php ~/backups/wiki/ccbwiki/LocalSettings.php; then
cp -p ./LocalSettings.php ~/backups/wiki/ccbwiki/
files_changed=1
fi
##- make the whole wiki read-only:
#nice -n 19 perl -i -pe 's/^[#\s]+(\$wgReadOnly[ =])/$1/' LocalSettings.php
#clean up pChart4mw cache files older than 20 days
cd uploads/pChart4mw_cache
cache_size=$(du -s | cut -f1)
if [[ $cache_size -gt 20000000 ]]; then
echo "Cleaning up cache.." > $ferr
( nice -n19 find . -maxdepth 1 -mtime +20 -exec /bin/rm -rf {} \; ) 2>>$ferr
fi
##### --- see if wiki files have been updated (uploads, extensions, configuration etc.)
cd $wikidir
ff=$bn.ccbwiki_files.tar.bz2
fsync=~/backups/wiki/$bn.rsync.info
(
nice -n19 rsync -Wai --delete --exclude=uploads/pChart4mw_cache/'*' \
--exclude=LocalSettings.php ./ ~/backups/wiki/ccbwiki > $fsync
) 2>>$ferr
if grep -q -E '^(>f|c)' $fsync 2>/dev/null; then
#we have changes, build the tar file
files_changed=1
#else
# rsync -aicI ./LocalSettings.php ~/backups/wiki/ccbwiki/LocalSettings.php > $fsync
# if grep -q '^>f' $fsync 2>/dev/null; then
# files_changed=1
# fi
fi
#( nice -n 19 tar cfj ~/backups/wiki/$bfiles uploads \
# images extensions LocalSettings.php ) 2>>$ferr
dbf="$bn.ccbwiki_db.sql.gz"
echo "Backing up database with mysqldump ($dbf).." >> $ferr
#using ~/.my.cnf instead, removed: -u "$dbuser" -p"$dbpass"
( nice -n 19 mysqldump -h igm5.pha.jhu.edu --default-character-set=binary \
--single-transaction igmwiki -c | nice -n 19 gzip -9 > ~/backups/wiki/$dbf
) 2>>$ferr
### -- xml dump:
## --disable for now
#xmlf="$bn.ccbwiki_xml.bz2"
#echo "Backing up pages as xml ($xmlf).." >> $ferr
#( nice -n19 php -d error_reporting=E_ERROR maintenance/dumpBackup.php --current | \
# nice -n19 bzip2 -9 > ~/backups/wiki/$xmlf
#) 2>>$ferr
##- restore wiki to read-write:
#nice -n 19 perl -i -pe 's/^\s*(\$wgReadOnly[ =])/## $1/' LocalSettings.php
##- prepare and send the backups to remote backup locations
cd ~/backups/wiki
prevff=$(ls -1t *.ccbwiki_files.tar.bz2 2>/dev/null | head -1)
if [[ -z $prevff ]]; then
files_changed=1
fi
if [[ $files_changed ]]; then
echo "Backing up files ($ff).." >> $ferr
#echo "Files changed, building $ff" >> $ferr
nice -n 19 tar cfj $ff ccbwiki 2>> $ferr
c=0
for fn in $( ls -1t *.ccbwiki_files.tar.bz2 2>/dev/null ) ; do
(( c++ ))
if [[ $c -gt 5 ]]; then
/bin/rm -f $fn
fi
done
else
if [[ $prevff ]]; then
touch $prevff #make it current
ff=$prevff
prevff=''
fi
fi
# clean up files older than 30 days ONLY IF these backup files are larger
dbfs=$(stat -c'%s' $dbf)
oldbf=$(ls -1t *.ccbwiki_db.sql.gz 2>/dev/null | tail -1)
oldbfs=$(stat -c'%s' $oldbf)
cleanup_old=1
(( oldbfs -= 50000 ))
#echo "Adjusted oldb file size: $oldbfs"
errtext=$(grep -E -i 'error|fail|cannot|space|couldn|fault' $ferr 2>/dev/null)
#echo "errtext='$errtext'"
if [[ $dbfs -lt 1024 || $dbfs -lt $oldbfs || $errtext ]]; then
cleanup_old=0
echo "Warning: issues encountered during last backup ($dbf, size $dbfs), see above." >> $ferr
oldmailer=$(mail -r 2>&1 | grep -F 'invalid option')
if [[ $oldmailer ]]; then
mail -s 'ccbwiki backup issue' '[email protected]' -- -f 'backuper@'$host.jhu.edu < $ferr
else
mail -s 'ccbwiki backup issue' -r 'backuper@'$host.jhu.edu '[email protected]' < $ferr
fi
fi
if [[ $cleanup_old -gt 0 ]]; then
#echo "Cleaning up older backups older than 30 days"
if [[ $prevff ]]; then
/bin/rm -f $prevff
fi
nice -n19 find . -maxdepth 1 -name 'bck_*.ccbwiki_*.*' -mtime +30 -exec /bin/rm -rf {} \;
fi
ferr=$bn.ssh.stderr
echo "" > $ferr
scperr=''
for rdest in "${rdirs[@]}" ; do
scp -q $dbf $rdest/ 2>>$ferr
if [[ $? != 0 ]]; then scperr=1; fi
#scp -q $xmlf $rdest/ 2>>$ferr
if [[ $? != 0 ]]; then scperr=1; fi
if [[ $files_changed ]]; then
scp -q $ff $rdest/ 2>>$ferr
if [[ $? != 0 ]]; then scperr=1; fi
fi
oifs=$IFS
IFS=':'
arr=($rdest)
rhost=${arr[0]}
rdir=${arr[1]}
#echo "Host is '$rhost', dir is '$rdir'"
rbasedir=${rdir%/*} #the remote "backups" directory
rtdir=${rdir##*/} #target subdirectory under the "backups" directory
# notify the remote host
#echo "running: ssh $rhost \"nohup $rbasedir/backup_received.sh '$rdir' '$bn' '$cleanup_old' '$ff' > /dev/null 2>&1\"" >> $ferr
echo "Notifying host $rhost .." >> $ferr
( ssh $rhost "nohup $rbasedir/backup_received.sh '$rdir' '$bn' '$cleanup_old' \
'$ff'" 2>&1
) 1>>$ferr
IFS=$oifs
done
err=$(grep -E -i 'error|fail|cannot|couldn|fault|timeout|discon' $ferr 2>/dev/null)
if [[ $err || $scperr ]]; then
oldmailer=$(mail -r 2>&1 | grep -F 'invalid option')
if [[ $oldmailer ]]; then
mail -s 'ccbwiki backup remote issue' '[email protected]' -- -f 'backuper@'$host.jhu.edu < $ferr
else
mail -s 'ccbwiki backup remote issue' -r 'backuper@'$host.jhu.edu '[email protected]' < $ferr
fi
fi