[2031] Workflows for database backup/restore (#1519)

DFE-Digital · Sep 18, 2024 · 0f29981 · 0f29981
1 parent 5365649
commit 0f29981
Show file tree

Hide file tree

Showing 5 changed files with 226 additions and 57 deletions.
diff --git a/.github/workflows/backup-db.yml b/.github/workflows/backup-db.yml
@@ -2,71 +2,92 @@ name: Backup production database
 
 on:
   workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to backup
+        required: true
+        default: tes
+        type: choice
+        options:
+        - test
+        - production
+      backupFileName:
+        description: Backup file name. Default is [SERVICE_NAME]_[CONFIG_SHORT]_adhoc_YYYY-MM-DD
+        required: false
+        type: string
+        default: default
+      backupPTRServer:
+        description: Set to true if backing up a point in time restored database server
+        required: false
+        type: boolean
+        default: false
   schedule:
     - cron: "0 4 * * *" # 04:00 UTC
 
+env:
+  SERVICE_NAME: trs
+  TF_VARS_PATH: terraform/aks/config
+
 jobs:
+
   backup:
     name: Backup database
     runs-on: ubuntu-latest
-
-    environment: production
-
+    environment:
+      name: ${{ inputs.environment || production }}
     env:
-      RESOURCE_GROUP: s189p01-trs-pd-rg
-      KEYVAULT_NAME: s189p01-trs-pd-inf-kv
-      CLUSTER_NAME: s189p01-tsc-production-aks
-      CLUSTER_RESOURCE_GROUP: s189p01-tsc-pd-rg
+      DEPLOY_ENV: ${{ inputs.environment || production  }}
+      BACKUP_FILE: ${{ inputs.backupFileName || schedule  }}
 
     steps:
-      - uses: actions/checkout@v4
-
-      - uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: 1.5.0
-          terraform_wrapper: false
-
-      - uses: Azure/login@v2
-        with:
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
-
-      - uses: DFE-Digital/github-actions/set-kubelogin-environment@master
-        with:
-          azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
-
-      - name: Install kubectl
-        uses: azure/setup-kubectl@v4
-
-      - name: Get k8s credentials
-        run: make ci production get-cluster-credentials
-
-      - name: Install konduit
-        run: make install-konduit
-
-      - name: Dump database
-        run: bin/konduit.sh -k ${{ env.KEYVAULT_NAME }} -d trs trs-production-worker -- pg_dump -E utf8 --compress=1 --clean --if-exists --no-owner --verbose -f backup.sql.gz
-
-      - name: Get backup storage account
-        id: azure-backup-storage
-        run: |
-          make ci production terraform-init
-          echo "account-name=$(terraform -chdir=terraform/aks output -raw postgres_azure_backup_storage_account_name)" >> $GITHUB_OUTPUT
-          echo "container-name=$(terraform -chdir=terraform/aks output -raw postgres_azure_backup_storage_container_name)" >> $GITHUB_OUTPUT
-
-      - name: Get storage account connection string
-        run: |
-          STORAGE_CONN_STR=$(az storage account show-connection-string -g ${{ env.RESOURCE_GROUP }} -n ${{ steps.azure-backup-storage.outputs.account-name }} --query 'connectionString')
-          echo "::add-mask::$STORAGE_CONN_STR"
-          echo "AZURE_STORAGE_CONNECTION_STRING=$STORAGE_CONN_STR" >> $GITHUB_ENV
-
-      - name: Upload backup
-        run: |
-          az config set extension.use_dynamic_install=yes_without_prompt
-          az config set core.only_show_errors=true
-          az storage azcopy blob upload \
-            --container ${{ steps.azure-backup-storage.outputs.container-name }} \
-            --source backup.sql.gz \
-            --destination $(date +"%F-%H").sql.gz
+    - uses: actions/checkout@v4
+
+    - name: Set environment variables
+      run: |
+        source global_config/${DEPLOY_ENV}.sh
+        tf_vars_file=${{ env.TF_VARS_PATH }}/${DEPLOY_ENV}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_NAME}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+        echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_NAME}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+        TODAY=$(date +"%F")
+        echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_NAME}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+        if [ "${{ env.BACKUP_FILE }}" == "schedule" ]; then
+          BACKUP_FILE=${SERVICE_NAME}_${CONFIG_SHORT}_${TODAY}
+        elif [ "${{ env.BACKUP_FILE }}" == "default" ]; then
+          BACKUP_FILE=${SERVICE_NAME}_${CONFIG_SHORT}_adhoc_${TODAY}
+        else
+          BACKUP_FILE=${{ env.BACKUP_FILE }}
+        fi
+        echo "BACKUP_FILE=$BACKUP_FILE" >> $GITHUB_ENV
+        echo "KEYVAULT_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-inf-kv" >> $GITHUB_ENV
+
+    - name: Set PTR variables
+      if: ${{ github.event.inputs.backupPTRServer == 'true' }}
+      run: |
+        echo "PTR_DB_SERVER=${{ env.DB_SERVER }}-ptr" >> $GITHUB_ENV
+        echo "BACKUP_FILE=${{ env.BACKUP_FILE }}-ptr" >> $GITHUB_ENV
+
+    - name: Backup ${{ env.DEPLOY_ENV }} postgres
+      uses: DFE-Digital/github-actions/backup-postgres@master
+      with:
+        storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+        resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+        app-name: ${{ env.SERVICE_NAME }}-${{ env.DEPLOY_ENV }}-api
+        cluster: ${{ env.CLUSTER }}
+        azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+        backup-file: ${{ env.BACKUP_FILE }}.sql
+        ptr-db-server-name: ${{ env.PTR_DB_SERVER }}
+
+    - name: Backup Summary
+      if: success()
+      run: |
+        NOW=$(TZ=Europe/London date +"%F %R")
+        echo 'BACKUP SUCCESSFUL!' >> $GITHUB_STEP_SUMMARY
+        echo '  ENV: ${{ env.DEPLOY_ENV }}' >> $GITHUB_STEP_SUMMARY
+        echo "  AT : ${NOW}" >> $GITHUB_STEP_SUMMARY
+        echo '  DB SERVER: ${{ env.PTR_DB_SERVER || env.DB_SERVER  }}' >> $GITHUB_STEP_SUMMARY
+        echo '  STORAGE ACCOUNT: ${{ env.STORAGE_ACCOUNT_NAME }}' >> $GITHUB_STEP_SUMMARY
+        echo '  FILENAME: ${{ env.BACKUP_FILE }}.sql.gz' >> $GITHUB_STEP_SUMMARY
 
       - name: Get Slack webhook
         uses: Azure/get-keyvault-secrets@v1

diff --git a/.github/workflows/deploy-preprod.yml b/.github/workflows/deploy-preprod.yml
@@ -12,7 +12,7 @@ on:
       docker_image:
         type: string
 
-concurrency: deploy_preprod
+concurrency: deploy_pre-production
 jobs:
   package:
     name: Package application

diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml
@@ -13,7 +13,7 @@ on:
         required: true
         type: string
 
-concurrency: deploy_prod
+concurrency: deploy_production
 jobs:
   deploy_aks:
     name: Deploy to AKS

diff --git a/.github/workflows/postgres-ptr.yml b/.github/workflows/postgres-ptr.yml
@@ -0,0 +1,69 @@
+name: Restore database from point in time to new database server
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to restore
+        required: true
+        default: test
+        type: choice
+        options:
+        - test
+        - production
+      confirm-production:
+        description: Must be set to true if restoring production
+        required: true
+        default: 'false'
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+      restore-time:
+        description: Restore point in time in UTC. e.g. 2024-07-24T06:00:00
+        type: string
+        required: true
+
+env:
+  SERVICE_SHORT: trs
+  TF_VARS_PATH: terraform/aks/config
+
+jobs:
+  ptr-restore:
+    name: PTR Restore AKS Database
+    if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && github.event.inputs.confirm-production == 'true' )  }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    concurrency: deploy_${{ inputs.environment }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set environment variables
+      run: |
+        source global_config/${{ inputs.environment }}.sh
+        tf_vars_file=${TF_VARS_PATH}/${{ inputs.environment }}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+        echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+
+    - name: Restore ${{ inputs.environment }} postgres
+      uses: DFE-Digital/github-actions/ptr-postgres@master
+      with:
+        resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+        source-server: ${{ env.DB_SERVER }}
+        new-server: ${{ env.DB_SERVER }}-ptr
+        restore-time: ${{ inputs.restore-time }}
+        cluster: ${{ env.CLUSTER }}
+        azure-credentials: ${{ secrets.AZURE_CREDENTIALS}}
+
+    - name: Restore Summary
+      if: success()
+      run: |
+        NOW=$(TZ=Europe/London date +"%F %R")
+        echo 'RESTORE SUCCESSFUL!' >> $GITHUB_STEP_SUMMARY
+        echo '  ENV: ${{ inputs.environment }}' >> $GITHUB_STEP_SUMMARY
+        echo "  AT : ${NOW}" >> $GITHUB_STEP_SUMMARY
+        echo '  SOURCE SERVER: ${{ env.DB_SERVER }}' >> $GITHUB_STEP_SUMMARY
+        echo '  RESTORED SERVER: ${{ env.DB_SERVER }}-ptr' >> $GITHUB_STEP_SUMMARY
+        echo '  RESTORE POINT: ${{ inputs.restore-time }} UTC' >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/postgres-restore.yml b/.github/workflows/postgres-restore.yml
@@ -0,0 +1,79 @@
+name: Restore database from Azure storage
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to restore
+        required: true
+        default: test
+        type: choice
+        options:
+        - test
+#        - production
+      confirm-production:
+        description: Must be set to true if restoring production
+        required: true
+        default: 'false'
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+      backup-file:
+        description: Name of the backup file in Azure storage. e.g. trs_prod_2024-08-09.tar.gz. The default value is today's scheduled backup.
+        type: string
+        required: false
+
+env:
+  SERVICE_NAME: trs
+  SERVICE_SHORT: trs
+  TF_VARS_PATH: terraform/aks/config
+
+jobs:
+  restore:
+    name: Restore AKS Database
+    if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && github.event.inputs.confirm-production == 'true' )  }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    concurrency: deploy_${{ inputs.environment }}
+
+    steps:
+    - uses: actions/checkout@v4
+      name: Checkout
+
+    - name: Set environment variables
+      run: |
+        source global_config/${{ inputs.environment }}.sh
+        tf_vars_file=${{ env.TF_VARS_PATH }}/${{ inputs.environment }}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+        echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+        echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+        TODAY=$(date +"%F")
+        echo "BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql" >> $GITHUB_ENV
+        if [ "${{ inputs.backup-file }}" != "" ]; then
+          BACKUP_FILE=${{ inputs.backup-file }}
+        else
+          BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql.gz
+        fi
+        echo "BACKUP_FILE=$BACKUP_FILE" >> $GITHUB_ENV
+
+    - name: Restore ${{ inputs.environment }} postgres
+      uses: DFE-Digital/github-actions/restore-postgres-backup@master
+      with:
+        storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+        resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+        app-name: ${{ env.SERVICE_NAME }}-${{ inputs.environment }}-api
+        cluster: ${{ env.CLUSTER }}
+        azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+        backup-file: ${{ env.BACKUP_FILE }}
+
+    - name: Restore Summary
+      if: success()
+      run: |
+        NOW=$(TZ=Europe/London date +"%F %R")
+        echo 'RESTORE SUCCESSFUL!' >> $GITHUB_STEP_SUMMARY
+        echo '  ENV: ${{ inputs.environment }}' >> $GITHUB_STEP_SUMMARY
+        echo "  AT : ${NOW}" >> $GITHUB_STEP_SUMMARY
+        echo '  DB SERVER: ${{ env.DB_SERVER }}' >> $GITHUB_STEP_SUMMARY
+        echo '  BACKUP FILE RESTORED: ${{ env.BACKUP_FILE }}' >> $GITHUB_STEP_SUMMARY