diff --git a/.github/workflows/backup-db.yml b/.github/workflows/backup-db.yml index 02222ec55..b37c3931a 100644 --- a/.github/workflows/backup-db.yml +++ b/.github/workflows/backup-db.yml @@ -2,71 +2,92 @@ name: Backup production database on: workflow_dispatch: + inputs: + environment: + description: Environment to backup + required: true + default: tes + type: choice + options: + - test + - production + backupFileName: + description: Backup file name. Default is [SERVICE_NAME]_[CONFIG_SHORT]_adhoc_YYYY-MM-DD + required: false + type: string + default: default + backupPTRServer: + description: Set to true if backing up a point in time restored database server + required: false + type: boolean + default: false schedule: - cron: "0 4 * * *" # 04:00 UTC +env: + SERVICE_NAME: trs + TF_VARS_PATH: terraform/aks/config + jobs: + backup: name: Backup database runs-on: ubuntu-latest - - environment: production - + environment: + name: ${{ inputs.environment || production }} env: - RESOURCE_GROUP: s189p01-trs-pd-rg - KEYVAULT_NAME: s189p01-trs-pd-inf-kv - CLUSTER_NAME: s189p01-tsc-production-aks - CLUSTER_RESOURCE_GROUP: s189p01-tsc-pd-rg + DEPLOY_ENV: ${{ inputs.environment || production }} + BACKUP_FILE: ${{ inputs.backupFileName || schedule }} steps: - - uses: actions/checkout@v4 - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.5.0 - terraform_wrapper: false - - - uses: Azure/login@v2 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - - - uses: DFE-Digital/github-actions/set-kubelogin-environment@master - with: - azure-credentials: ${{ secrets.AZURE_CREDENTIALS }} - - - name: Install kubectl - uses: azure/setup-kubectl@v4 - - - name: Get k8s credentials - run: make ci production get-cluster-credentials - - - name: Install konduit - run: make install-konduit - - - name: Dump database - run: bin/konduit.sh -k ${{ env.KEYVAULT_NAME }} -d trs trs-production-worker -- pg_dump -E utf8 --compress=1 --clean --if-exists --no-owner --verbose -f backup.sql.gz - - - name: Get backup storage account - id: azure-backup-storage - run: | - make ci production terraform-init - echo "account-name=$(terraform -chdir=terraform/aks output -raw postgres_azure_backup_storage_account_name)" >> $GITHUB_OUTPUT - echo "container-name=$(terraform -chdir=terraform/aks output -raw postgres_azure_backup_storage_container_name)" >> $GITHUB_OUTPUT - - - name: Get storage account connection string - run: | - STORAGE_CONN_STR=$(az storage account show-connection-string -g ${{ env.RESOURCE_GROUP }} -n ${{ steps.azure-backup-storage.outputs.account-name }} --query 'connectionString') - echo "::add-mask::$STORAGE_CONN_STR" - echo "AZURE_STORAGE_CONNECTION_STRING=$STORAGE_CONN_STR" >> $GITHUB_ENV - - - name: Upload backup - run: | - az config set extension.use_dynamic_install=yes_without_prompt - az config set core.only_show_errors=true - az storage azcopy blob upload \ - --container ${{ steps.azure-backup-storage.outputs.container-name }} \ - --source backup.sql.gz \ - --destination $(date +"%F-%H").sql.gz + - uses: actions/checkout@v4 + + - name: Set environment variables + run: | + source global_config/${DEPLOY_ENV}.sh + tf_vars_file=${{ env.TF_VARS_PATH }}/${DEPLOY_ENV}.tfvars.json + echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV + echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_NAME}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV + echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_NAME}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV + TODAY=$(date +"%F") + echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_NAME}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV + if [ "${{ env.BACKUP_FILE }}" == "schedule" ]; then + BACKUP_FILE=${SERVICE_NAME}_${CONFIG_SHORT}_${TODAY} + elif [ "${{ env.BACKUP_FILE }}" == "default" ]; then + BACKUP_FILE=${SERVICE_NAME}_${CONFIG_SHORT}_adhoc_${TODAY} + else + BACKUP_FILE=${{ env.BACKUP_FILE }} + fi + echo "BACKUP_FILE=$BACKUP_FILE" >> $GITHUB_ENV + echo "KEYVAULT_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-inf-kv" >> $GITHUB_ENV + + - name: Set PTR variables + if: ${{ github.event.inputs.backupPTRServer == 'true' }} + run: | + echo "PTR_DB_SERVER=${{ env.DB_SERVER }}-ptr" >> $GITHUB_ENV + echo "BACKUP_FILE=${{ env.BACKUP_FILE }}-ptr" >> $GITHUB_ENV + + - name: Backup ${{ env.DEPLOY_ENV }} postgres + uses: DFE-Digital/github-actions/backup-postgres@master + with: + storage-account: ${{ env.STORAGE_ACCOUNT_NAME }} + resource-group: ${{ env.RESOURCE_GROUP_NAME }} + app-name: ${{ env.SERVICE_NAME }}-${{ env.DEPLOY_ENV }}-api + cluster: ${{ env.CLUSTER }} + azure-credentials: ${{ secrets.AZURE_CREDENTIALS }} + backup-file: ${{ env.BACKUP_FILE }}.sql + ptr-db-server-name: ${{ env.PTR_DB_SERVER }} + + - name: Backup Summary + if: success() + run: | + NOW=$(TZ=Europe/London date +"%F %R") + echo 'BACKUP SUCCESSFUL!' >> $GITHUB_STEP_SUMMARY + echo ' ENV: ${{ env.DEPLOY_ENV }}' >> $GITHUB_STEP_SUMMARY + echo " AT : ${NOW}" >> $GITHUB_STEP_SUMMARY + echo ' DB SERVER: ${{ env.PTR_DB_SERVER || env.DB_SERVER }}' >> $GITHUB_STEP_SUMMARY + echo ' STORAGE ACCOUNT: ${{ env.STORAGE_ACCOUNT_NAME }}' >> $GITHUB_STEP_SUMMARY + echo ' FILENAME: ${{ env.BACKUP_FILE }}.sql.gz' >> $GITHUB_STEP_SUMMARY - name: Get Slack webhook uses: Azure/get-keyvault-secrets@v1 diff --git a/.github/workflows/deploy-preprod.yml b/.github/workflows/deploy-preprod.yml index 70e1e3fec..c802d6960 100644 --- a/.github/workflows/deploy-preprod.yml +++ b/.github/workflows/deploy-preprod.yml @@ -12,7 +12,7 @@ on: docker_image: type: string -concurrency: deploy_preprod +concurrency: deploy_pre-production jobs: package: name: Package application diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index caf41512b..29eaf89e8 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -13,7 +13,7 @@ on: required: true type: string -concurrency: deploy_prod +concurrency: deploy_production jobs: deploy_aks: name: Deploy to AKS diff --git a/.github/workflows/postgres-ptr.yml b/.github/workflows/postgres-ptr.yml new file mode 100644 index 000000000..789f4eeb6 --- /dev/null +++ b/.github/workflows/postgres-ptr.yml @@ -0,0 +1,69 @@ +name: Restore database from point in time to new database server + +on: + workflow_dispatch: + inputs: + environment: + description: Environment to restore + required: true + default: test + type: choice + options: + - test + - production + confirm-production: + description: Must be set to true if restoring production + required: true + default: 'false' + type: choice + options: + - 'false' + - 'true' + restore-time: + description: Restore point in time in UTC. e.g. 2024-07-24T06:00:00 + type: string + required: true + +env: + SERVICE_SHORT: trs + TF_VARS_PATH: terraform/aks/config + +jobs: + ptr-restore: + name: PTR Restore AKS Database + if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && github.event.inputs.confirm-production == 'true' ) }} + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + concurrency: deploy_${{ inputs.environment }} + + steps: + - uses: actions/checkout@v4 + + - name: Set environment variables + run: | + source global_config/${{ inputs.environment }}.sh + tf_vars_file=${TF_VARS_PATH}/${{ inputs.environment }}.tfvars.json + echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV + echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV + echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV + + - name: Restore ${{ inputs.environment }} postgres + uses: DFE-Digital/github-actions/ptr-postgres@master + with: + resource-group: ${{ env.RESOURCE_GROUP_NAME }} + source-server: ${{ env.DB_SERVER }} + new-server: ${{ env.DB_SERVER }}-ptr + restore-time: ${{ inputs.restore-time }} + cluster: ${{ env.CLUSTER }} + azure-credentials: ${{ secrets.AZURE_CREDENTIALS}} + + - name: Restore Summary + if: success() + run: | + NOW=$(TZ=Europe/London date +"%F %R") + echo 'RESTORE SUCCESSFUL!' >> $GITHUB_STEP_SUMMARY + echo ' ENV: ${{ inputs.environment }}' >> $GITHUB_STEP_SUMMARY + echo " AT : ${NOW}" >> $GITHUB_STEP_SUMMARY + echo ' SOURCE SERVER: ${{ env.DB_SERVER }}' >> $GITHUB_STEP_SUMMARY + echo ' RESTORED SERVER: ${{ env.DB_SERVER }}-ptr' >> $GITHUB_STEP_SUMMARY + echo ' RESTORE POINT: ${{ inputs.restore-time }} UTC' >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/postgres-restore.yml b/.github/workflows/postgres-restore.yml new file mode 100644 index 000000000..bd9f25ea7 --- /dev/null +++ b/.github/workflows/postgres-restore.yml @@ -0,0 +1,79 @@ +name: Restore database from Azure storage + +on: + workflow_dispatch: + inputs: + environment: + description: Environment to restore + required: true + default: test + type: choice + options: + - test +# - production + confirm-production: + description: Must be set to true if restoring production + required: true + default: 'false' + type: choice + options: + - 'false' + - 'true' + backup-file: + description: Name of the backup file in Azure storage. e.g. trs_prod_2024-08-09.tar.gz. The default value is today's scheduled backup. + type: string + required: false + +env: + SERVICE_NAME: trs + SERVICE_SHORT: trs + TF_VARS_PATH: terraform/aks/config + +jobs: + restore: + name: Restore AKS Database + if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && github.event.inputs.confirm-production == 'true' ) }} + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + concurrency: deploy_${{ inputs.environment }} + + steps: + - uses: actions/checkout@v4 + name: Checkout + + - name: Set environment variables + run: | + source global_config/${{ inputs.environment }}.sh + tf_vars_file=${{ env.TF_VARS_PATH }}/${{ inputs.environment }}.tfvars.json + echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV + echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV + echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV + echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV + TODAY=$(date +"%F") + echo "BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql" >> $GITHUB_ENV + if [ "${{ inputs.backup-file }}" != "" ]; then + BACKUP_FILE=${{ inputs.backup-file }} + else + BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql.gz + fi + echo "BACKUP_FILE=$BACKUP_FILE" >> $GITHUB_ENV + + - name: Restore ${{ inputs.environment }} postgres + uses: DFE-Digital/github-actions/restore-postgres-backup@master + with: + storage-account: ${{ env.STORAGE_ACCOUNT_NAME }} + resource-group: ${{ env.RESOURCE_GROUP_NAME }} + app-name: ${{ env.SERVICE_NAME }}-${{ inputs.environment }}-api + cluster: ${{ env.CLUSTER }} + azure-credentials: ${{ secrets.AZURE_CREDENTIALS }} + backup-file: ${{ env.BACKUP_FILE }} + + - name: Restore Summary + if: success() + run: | + NOW=$(TZ=Europe/London date +"%F %R") + echo 'RESTORE SUCCESSFUL!' >> $GITHUB_STEP_SUMMARY + echo ' ENV: ${{ inputs.environment }}' >> $GITHUB_STEP_SUMMARY + echo " AT : ${NOW}" >> $GITHUB_STEP_SUMMARY + echo ' DB SERVER: ${{ env.DB_SERVER }}' >> $GITHUB_STEP_SUMMARY + echo ' BACKUP FILE RESTORED: ${{ env.BACKUP_FILE }}' >> $GITHUB_STEP_SUMMARY