From 1f384232edceb0f5acce7d0ed7eac5d91962289a Mon Sep 17 00:00:00 2001 From: John Seekins Date: Tue, 15 Aug 2023 06:53:47 -0600 Subject: [PATCH 1/3] remove full backup (we have snapshots) Signed-off-by: John Seekins --- docker/cron/backup.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/cron/backup.sh b/docker/cron/backup.sh index 10234fb7..0d9480a4 100755 --- a/docker/cron/backup.sh +++ b/docker/cron/backup.sh @@ -12,11 +12,11 @@ set -e #### # backup everything to private archive -echo "Extracting full backup..." -pg_dump -Fc openstatesorg > openstatesorg.pgdump -echo "Shipping full backup to s3" -aws s3 cp openstatesorg.pgdump "s3://openstates-backups/full-backup/$(date +%Y-%m-%d)-openstatesorg.pgdump" > /dev/null -rm -f openstatesorg.pgdump +#echo "Extracting full backup..." +#pg_dump -Fc openstatesorg > openstatesorg.pgdump +#echo "Shipping full backup to s3" +#aws s3 cp openstatesorg.pgdump "s3://openstates-backups/full-backup/$(date +%Y-%m-%d)-openstatesorg.pgdump" > /dev/null +#rm -f openstatesorg.pgdump # layered approach for public echo "Executing public schema-only backup..." @@ -34,7 +34,8 @@ pg_dump -Fc openstatesorg --data-only \ echo "Uploading public backups to s3..." aws s3 cp --acl public-read public.pgdump "s3://data.openstates.org/postgres/daily/$(date +%Y-%m-%d)-public.pgdump" > /dev/null -aws s3 cp --acl public-read public.pgdump "s3://data.openstates.org/postgres/monthly/$(date +%Y-%m)-public.pgdump" > /dev/null +# only upload monthly dump on the first of the month +[[ $(date +%d) -eq 1 ]] && aws s3 cp --acl public-read public.pgdump "s3://data.openstates.org/postgres/monthly/$(date +%Y-%m)-public.pgdump" > /dev/null rm -f public.pgdump #echo "Extracting geo backup..." From a244b803148b3f400d1ce99e4dae8b17b17ce5e7 Mon Sep 17 00:00:00 2001 From: John Seekins Date: Tue, 15 Aug 2023 06:57:56 -0600 Subject: [PATCH 2/3] stick with sh compliant steps Signed-off-by: John Seekins --- docker/cron/backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cron/backup.sh b/docker/cron/backup.sh index 0d9480a4..8b2803da 100755 --- a/docker/cron/backup.sh +++ b/docker/cron/backup.sh @@ -35,7 +35,7 @@ pg_dump -Fc openstatesorg --data-only \ echo "Uploading public backups to s3..." aws s3 cp --acl public-read public.pgdump "s3://data.openstates.org/postgres/daily/$(date +%Y-%m-%d)-public.pgdump" > /dev/null # only upload monthly dump on the first of the month -[[ $(date +%d) -eq 1 ]] && aws s3 cp --acl public-read public.pgdump "s3://data.openstates.org/postgres/monthly/$(date +%Y-%m)-public.pgdump" > /dev/null +[ "$(date +%d)" -eq 1 ] && aws s3 cp --acl public-read public.pgdump "s3://data.openstates.org/postgres/monthly/$(date +%Y-%m)-public.pgdump" > /dev/null rm -f public.pgdump #echo "Extracting geo backup..." From 96634d7e6835263ac3fcd1899a33787d6bad5066 Mon Sep 17 00:00:00 2001 From: John Seekins Date: Tue, 15 Aug 2023 09:01:39 -0600 Subject: [PATCH 3/3] add some comments about disabled steps Signed-off-by: John Seekins --- docker/cron/backup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/cron/backup.sh b/docker/cron/backup.sh index 8b2803da..bde2fed4 100755 --- a/docker/cron/backup.sh +++ b/docker/cron/backup.sh @@ -12,6 +12,7 @@ set -e #### # backup everything to private archive +# Disabled because we now rely on snapshots from RDS #echo "Extracting full backup..." #pg_dump -Fc openstatesorg > openstatesorg.pgdump #echo "Shipping full backup to s3" @@ -38,6 +39,7 @@ aws s3 cp --acl public-read public.pgdump "s3://data.openstates.org/postgres/dai [ "$(date +%d)" -eq 1 ] && aws s3 cp --acl public-read public.pgdump "s3://data.openstates.org/postgres/monthly/$(date +%Y-%m)-public.pgdump" > /dev/null rm -f public.pgdump +# Currently disabled because it requires different credentials #echo "Extracting geo backup..." #pg_dump -Fc geo > openstates-geo.pgdump #echo "Shipping full backup to s3"