From b28243eeaf3330c73521a641042ab95f5e0b324f Mon Sep 17 00:00:00 2001 From: RMcVelia Date: Wed, 27 Nov 2024 16:44:07 +0000 Subject: [PATCH] Use azure storage account for all backups --- .github/actions/restore/action.yml | 56 +++++++++++++++++--------- .github/workflows/database-restore.yml | 20 +++++---- guides/setup-development.md | 4 +- 3 files changed, 51 insertions(+), 29 deletions(-) diff --git a/.github/actions/restore/action.yml b/.github/actions/restore/action.yml index 82e770c0c4..edc5a5565f 100644 --- a/.github/actions/restore/action.yml +++ b/.github/actions/restore/action.yml @@ -4,6 +4,8 @@ description: Restore AKS environment database inputs: azure-credentials: required: true + prod-credentials: + required: true environment: description: Environment to restore to required: true @@ -19,6 +21,35 @@ inputs: runs: using: composite steps: + - name: Set env variable + shell: bash + run: echo "SANITISED_FILE_NAME=publish_sanitised_$(date +"%F")" >> $GITHUB_ENV + + - name: Set KV environment variables + shell: bash + run: | + tf_vars_file=terraform/aks/workspace_variables/production.tfvars.json + echo "key_vault_name=$(jq -r '.key_vault_name' ${tf_vars_file})" >> $GITHUB_ENV + + - uses: azure/login@v2 + with: + creds: ${{ inputs.prod-credentials }} + + - name: Set Connection String + shell: bash + run: | + STORAGE_CONN_STR="$(az keyvault secret show --name PUBLISH-STORAGE-ACCOUNT-CONNECTION-STRING-PRODUCTION --vault-name ${{ env.key_vault_name }} | jq -r .value)" + echo "::add-mask::$STORAGE_CONN_STR" + echo "STORAGE_CONN_STR=$STORAGE_CONN_STR" >> $GITHUB_ENV + + - name: Download Backup + shell: bash + run: | + az storage blob download --container-name database-backup \ + --file ${SANITISED_FILE_NAME}.sql.gz --name ${SANITISED_FILE_NAME}.sql.gz \ + --connection-string '${{ env.STORAGE_CONN_STR }}' + az logout + - name: Set environment variables for AKS shell: bash run: | @@ -26,10 +57,6 @@ runs: echo "cluster=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV echo "app_environment=$(jq -r '.app_environment' ${tf_vars_file})" >> $GITHUB_ENV - - uses: azure/login@v2 - with: - creds: ${{ inputs.azure-credentials }} - - name: Set app name variable shell: bash run: | @@ -41,6 +68,10 @@ runs: echo "app_name=publish-${{ env.app_environment }}" >> $GITHUB_ENV fi + - uses: azure/login@v2 + with: + creds: ${{ inputs.azure-credentials }} + - name: Install kubectl if: ${{ env.app_environment != 'review' }} uses: DFE-Digital/github-actions/set-kubectl@master @@ -56,22 +87,7 @@ runs: make ci ${{ inputs.environment }} get-cluster-credentials make install-konduit - - name: Download Sanitised Backup - if: ${{ env.app_environment != 'review' }} - uses: actions/download-artifact@v4 - with: - name: backup_sanitised - - - name: Download Latest Sanitised Backup - if: ${{ env.app_environment == 'review' }} - shell: bash - run: | - sanitised_backup_workflow_run_id=$(gh run list -w "Database Backup and Restore" -s success --json databaseId --jq '.[].databaseId' -L 1) - echo "Download artifact for Database Backup and Restore workflow run ID: $sanitised_backup_workflow_run_id" - gh run download $sanitised_backup_workflow_run_id - mv backup_sanitised/backup_sanitised.sql ./ - - name: Restore backup to aks env database shell: bash run: | - bin/konduit.sh -i backup_sanitised.sql -t 7200 ${{ env.app_name }} -- psql + bin/konduit.sh -i ${SANITISED_FILE_NAME}.sql.gz -c -t 7200 ${{ env.app_name }} -- psql diff --git a/.github/workflows/database-restore.yml b/.github/workflows/database-restore.yml index d8d4b439f5..2c4eb5b169 100644 --- a/.github/workflows/database-restore.yml +++ b/.github/workflows/database-restore.yml @@ -70,7 +70,10 @@ jobs: run: | az storage blob upload --container-name database-backup \ --file ${PROD_BACKUP}.tar.gz --name ${PROD_BACKUP}.tar.gz \ - --connection-string '${{ env.STORAGE_CONN_STR }}' + --connection-string '${{ env.STORAGE_CONN_STR }}' --overwrite + + - name: Set env variable + run: echo "SANITISED_FILE_NAME=publish_sanitised_$(date +"%F")" >> $GITHUB_ENV - name: Sanitise the Database backup run: | @@ -90,7 +93,7 @@ jobs: rm ${{ env.PROD_BACKUP }} echo "::group::Backup Sanitised Database" - pg_dump --encoding utf8 --clean --no-owner --if-exists -d ${DATABASE_NAME} -f backup_sanitised.sql + pg_dump --compress=1 --encoding utf8 --clean --no-owner --if-exists -d ${DATABASE_NAME} -f ${SANITISED_FILE_NAME}.sql.gz echo "::endgroup::" env: DATABASE_NAME: teacher_training_api @@ -99,12 +102,12 @@ jobs: PGHOST: localhost PGPORT: 5432 - - name: Upload Sanitised Backup - uses: actions/upload-artifact@v4 - with: - name: backup_sanitised - path: backup_sanitised.sql - retention-days: 7 + - name: Upload Backup to Azure Storage + run: | + az storage blob upload --container-name database-backup \ + --file ${SANITISED_FILE_NAME}.sql.gz --name ${SANITISED_FILE_NAME}.sql.gz --overwrite \ + --connection-string '${{ env.STORAGE_CONN_STR }}' + rm ${SANITISED_FILE_NAME}.sql.gz restore: needs: [backup] @@ -124,4 +127,5 @@ jobs: uses: ./.github/actions/restore/ with: azure-credentials: ${{ secrets[format('AZURE_CREDENTIALS_{0}', matrix.environment)] }} + prod-credentials: ${{ secrets.AZURE_CREDENTIALS_PRODUCTION }} environment: ${{ matrix.environment }} diff --git a/guides/setup-development.md b/guides/setup-development.md index 11e66de304..6895647d69 100644 --- a/guides/setup-development.md +++ b/guides/setup-development.md @@ -69,7 +69,9 @@ The commands from the previous section will seed the database with some test dat To seed the database with a sanitised production dump: -- Download the sanitised production dump from the [Github Actions page](https://github.com/DFE-Digital/publish-teacher-training/actions/workflows/database-restore.yml) and download the latest successful run. +- Download the sanitised production dump from the Azure Storage Account. +- In the Azure portal, go to 'Storage Accounts' -> 's189p01pttdbbkppdsa' -> 'Containers' -> 'database-backup' +- Download the latest sanitised backup. - Unzip the file and you should see a file called `backup_sanitised.sql`. Then run the following command to populate the database: