Use azure storage account for all backups

.github/actions/restore/action.yml

@@ -4,6 +4,8 @@ description: Restore AKS environment database
 inputs:
   azure-credentials:
     required: true
+  prod-credentials:
+    required: true
   environment:
     description: Environment to restore to
     required: true
@@ -19,17 +21,42 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: Set env variable
+      shell: bash
+      run: echo "SANITISED_FILE_NAME=publish_sanitised_$(date +"%F")" >> $GITHUB_ENV
+
+    - name: Set KV environment variables
+      shell: bash
+      run: |
+        tf_vars_file=terraform/aks/workspace_variables/production.tfvars.json
+        echo "key_vault_name=$(jq -r '.key_vault_name' ${tf_vars_file})" >> $GITHUB_ENV
+
+    - uses: azure/login@v2
+      with:
+        creds: ${{ inputs.prod-credentials }}
+
+    - name: Set Connection String
+      shell: bash
+      run: |
+        STORAGE_CONN_STR="$(az keyvault secret show --name PUBLISH-STORAGE-ACCOUNT-CONNECTION-STRING-PRODUCTION --vault-name ${{ env.key_vault_name }} | jq -r .value)"
+        echo "::add-mask::$STORAGE_CONN_STR"
+        echo "STORAGE_CONN_STR=$STORAGE_CONN_STR" >> $GITHUB_ENV
+
+    - name: Download Backup
+      shell: bash
+      run: |
+        az storage blob download --container-name database-backup \
+        --file ${SANITISED_FILE_NAME}.sql.gz --name ${SANITISED_FILE_NAME}.sql.gz \
+        --connection-string '${{ env.STORAGE_CONN_STR }}'
+        az logout
+
     - name: Set environment variables for AKS
       shell: bash
       run: |
         tf_vars_file=terraform/aks/workspace_variables/${{ inputs.environment }}.tfvars.json
         echo "cluster=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
         echo "app_environment=$(jq -r '.app_environment' ${tf_vars_file})" >> $GITHUB_ENV
 
-    - uses: azure/login@v2
-      with:
-        creds: ${{ inputs.azure-credentials }}
-
     - name: Set app name variable
       shell: bash
       run: |
@@ -41,6 +68,10 @@ runs:
           echo "app_name=publish-${{ env.app_environment }}" >> $GITHUB_ENV
         fi
 
+    - uses: azure/login@v2
+      with:
+        creds: ${{ inputs.azure-credentials }}
+
     - name: Install kubectl
       if: ${{ env.app_environment != 'review' }}
       uses: DFE-Digital/github-actions/set-kubectl@master
@@ -56,22 +87,7 @@ runs:
         make ci ${{ inputs.environment }} get-cluster-credentials
         make install-konduit
 
-    - name: Download Sanitised Backup
-      if: ${{ env.app_environment != 'review' }}
-      uses: actions/download-artifact@v4
-      with:
-        name: backup_sanitised
-
-    - name: Download Latest Sanitised Backup
-      if: ${{ env.app_environment == 'review' }}
-      shell: bash
-      run: |
-        sanitised_backup_workflow_run_id=$(gh run list -w "Database Backup and Restore" -s success --json databaseId --jq '.[].databaseId' -L 1)
-        echo "Download artifact for Database Backup and Restore workflow run ID: $sanitised_backup_workflow_run_id"
-        gh run download $sanitised_backup_workflow_run_id
-        mv backup_sanitised/backup_sanitised.sql ./
-
     - name: Restore backup to aks env database
       shell: bash
       run: |
-        bin/konduit.sh -i backup_sanitised.sql -t 7200 ${{ env.app_name }} -- psql
+        bin/konduit.sh -i ${SANITISED_FILE_NAME}.sql.gz -c -t 7200 ${{ env.app_name }} -- psql

.github/workflows/database-restore.yml

@@ -70,7 +70,10 @@ jobs:
         run: |
           az storage blob upload --container-name database-backup \
           --file ${PROD_BACKUP}.tar.gz --name ${PROD_BACKUP}.tar.gz \
-          --connection-string '${{ env.STORAGE_CONN_STR }}'
+          --connection-string '${{ env.STORAGE_CONN_STR }}' --overwrite
+
+      - name: Set env variable
+        run: echo "SANITISED_FILE_NAME=publish_sanitised_$(date +"%F")" >> $GITHUB_ENV
 
       - name: Sanitise the Database backup
         run: |
@@ -90,7 +93,7 @@ jobs:
           rm ${{ env.PROD_BACKUP }}
 
           echo "::group::Backup Sanitised Database"
-          pg_dump --encoding utf8 --clean --no-owner --if-exists -d ${DATABASE_NAME} -f backup_sanitised.sql
+          pg_dump --compress=1 --encoding utf8 --clean --no-owner --if-exists -d ${DATABASE_NAME} -f ${SANITISED_FILE_NAME}.sql.gz
           echo "::endgroup::"
         env:
           DATABASE_NAME: teacher_training_api
@@ -99,12 +102,12 @@ jobs:
           PGHOST: localhost
           PGPORT: 5432
 
-      - name: Upload Sanitised Backup
-        uses: actions/upload-artifact@v4
-        with:
-          name: backup_sanitised
-          path: backup_sanitised.sql
-          retention-days: 7
+      - name: Upload Backup to Azure Storage
+        run: |
+          az storage blob upload --container-name database-backup \
+          --file ${SANITISED_FILE_NAME}.sql.gz --name ${SANITISED_FILE_NAME}.sql.gz --overwrite \
+          --connection-string '${{ env.STORAGE_CONN_STR }}'
+          rm ${SANITISED_FILE_NAME}.sql.gz
 
   restore:
     needs: [backup]
@@ -124,4 +127,5 @@ jobs:
         uses: ./.github/actions/restore/
         with:
           azure-credentials: ${{ secrets[format('AZURE_CREDENTIALS_{0}', matrix.environment)] }}
+          prod-credentials: ${{ secrets.AZURE_CREDENTIALS_PRODUCTION }}
           environment: ${{ matrix.environment }}

guides/setup-development.md

@@ -69,7 +69,9 @@ The commands from the previous section will seed the database with some test dat
 
 To seed the database with a sanitised production dump:
 
-- Download the sanitised production dump from the [Github Actions page](https://github.com/DFE-Digital/publish-teacher-training/actions/workflows/database-restore.yml) and download the latest successful run.
+- Download the sanitised production dump from the Azure Storage Account.
+- In the Azure portal, go to 'Storage Accounts' -> 's189p01pttdbbkppdsa' -> 'Containers' -> 'database-backup'
+- Download the latest sanitised backup.
 - Unzip the file and you should see a file called `backup_sanitised.sql`.
 
 Then run the following command to populate the database: