Add database backup workflow for aks

DFE-Digital · Oct 26, 2023 · 0857131 · 0857131
1 parent 5db1569
commit 0857131
Show file tree

Hide file tree

Showing 7 changed files with 214 additions and 1 deletion.
diff --git a/.github/workflows/actions/database-backup/action.yml b/.github/workflows/actions/database-backup/action.yml
@@ -0,0 +1,162 @@
+name: Backup AKS Database
+description: backs up AKS database to Azure Storage
+
+inputs:
+  environment:
+    description: "The name of the environment"
+    required: true
+  azure_credentials:
+    description: "JSON object containing a service principal that can read from Azure Key Vault"
+    required: true
+
+outputs:
+  backup_artifact:
+    description: "The backup artifact name"
+    value: ${{ inputs.environment }}-backup
+
+runs:
+  using: composite
+
+  steps:
+    - uses: actions/checkout@v4
+      id: Checkout
+
+    - name: Prepare application environment
+      uses: ./.github/actions/prepare-app-env
+
+    - name: Set KV environment variables
+      shell: bash
+      id: set_kv_env_vars
+      run: |
+        tf_vars_file=terraform/aks/workspace_variables/${{ inputs.environment }}_aks.tfvars.json
+        echo "$PWD"
+        cat $tf_vars_file
+        INF_VAULT_NAME=$(jq -r '.inf_vault_name' ${tf_vars_file})
+        NAMESPACE=$(jq -r '.namespace' ${tf_vars_file})
+        APP_ENVIRONMENT=$(jq -r '.app_environment' ${tf_vars_file})
+        CLUSTER=$(jq -r '.cluster' ${tf_vars_file})
+
+        echo "INF_VAULT_NAME=$INF_VAULT_NAME" >> $GITHUB_ENV
+        echo "NAMESPACE=$NAMESPACE" >> $GITHUB_ENV
+        echo "CLUSTER=$CLUSTER" >> $GITHUB_ENV
+        echo "APP_ENVIRONMENT=$APP_ENVIRONMENT" >> $GITHUB_ENV
+
+        echo "envs are:  ${{ env.INF_VAULT_NAME }} ${{ env.NAMESPACE }}  ${{ env.APP_ENVIRONMENT }}  ${{ env.CLUSTER }}"
+
+    - uses: Azure/login@v1
+      with:
+        creds: ${{ inputs.azure_credentials }}
+
+    - name: Fetch slack web hook
+      uses: azure/CLI@v1
+      id: slack-web-hook
+      with:
+        inlineScript: |
+          SECRET_VALUE=$(az keyvault secret show --name "SLACK-WEBHOOK" --vault-name "${{ env.INF_VAULT_NAME }}" --query "value" -o tsv)
+          echo "::add-mask::$SECRET_VALUE"
+          echo "SLACK-WEBHOOK=$SECRET_VALUE" >> $GITHUB_OUTPUT
+
+    - name: Install kubectl
+      uses: azure/setup-kubectl@v3
+      with:
+        version: "v1.26.1"
+
+    - name: Get environment variables
+      shell: bash
+      run: |
+        test_cluster_rg=s189t01-tsc-ts-rg
+        test_cluster_name=s189t01-tsc-test-aks
+        production_cluster_rg=s189p01-tsc-pd-rg
+        production_cluster_name=s189p01-tsc-production-aks
+        BACKUP_FILE_NAME=faltrn_${{ inputs.environment }}_$(date +"%F")
+        echo "BACKUP_FILE_NAME=$BACKUP_FILE_NAME" >> $GITHUB_ENV
+
+        case "${{ inputs.environment }}" in
+        development)
+            echo "cluster_rg=$test_cluster_rg" >> $GITHUB_ENV
+            echo "cluster_name=$test_cluster_name" >> $GITHUB_ENV
+            echo "app_name=find-a-lost-trn-development" >> $GITHUB_ENV
+            echo "storage_account=s189t01faltrndbbkpdvsa" >> $GITHUB_ENV
+            ;;
+        test)
+            echo "cluster_rg=$test_cluster_rg" >> $GITHUB_ENV
+            echo "cluster_name=$test_cluster_name" >> $GITHUB_ENV
+            echo "app_name=find-a-lost-trn-test" >> $GITHUB_ENV
+            echo "storage_account=s189t01faltrndbbkptssa" >> $GITHUB_ENV
+            ;;
+        preproduction)
+            echo "cluster_rg=$test_cluster_rg" >> $GITHUB_ENV
+            echo "cluster_name=$test_cluster_name" >> $GITHUB_ENV
+            echo "app_name=find-a-lost-trn-preproduction" >> $GITHUB_ENV
+            echo "storage_account=s189t01faltrndbbkpppsa" >> $GITHUB_ENV
+            ;;
+        production)
+            echo "cluster_rg=$production_cluster_rg" >> $GITHUB_ENV
+            echo "cluster_name=$production_cluster_name" >> $GITHUB_ENV
+            echo "app_name=find-a-lost-trn-production" >> $GITHUB_ENV
+            echo "storage_account=s189p01faltrndbbkppdsa" >> $GITHUB_ENV
+            ;;
+        *)
+            echo "unknown cluster"
+            ;;
+        esac
+
+    - uses: azure/setup-kubectl@v3
+
+    - name: K8 setup
+      shell: bash
+      run: |
+        az aks get-credentials -g ${{ env.cluster_rg }} -n ${{ env.cluster_name }}
+        make bin/konduit.sh
+
+    - name: Setup postgres client
+      uses: DFE-Digital/github-actions/install-postgres-client@master
+
+    - name: Set environment variable
+      shell: bash
+      run: |
+        BACKUP_FILE_NAME=faltrn_${{ inputs.environment }}_$(date +"%F")
+
+    - name: Backup ${{ inputs.environment }} DB
+      shell: bash
+      run: |
+        bin/konduit.sh find-a-lost-trn-${{ inputs.environment }} -- pg_dump -E utf8 --clean --if-exists --no-owner --verbose --no-password -f ${BACKUP_FILE_NAME}.sql
+        tar -cvzf ${BACKUP_FILE_NAME}.tar.gz ${BACKUP_FILE_NAME}.sql
+
+    - name: Set Connection String
+      shell: bash
+      run: |
+        STORAGE_CONN_STR="$(az keyvault secret show --name FALTRN-BACKUP-STORAGE-CONNECTION-STRING-AKS --vault-name ${{ env.INF_VAULT_NAME }} | jq -r .value)"
+        echo "::add-mask::$STORAGE_CONN_STR"
+        echo "STORAGE_CONN_STR=$STORAGE_CONN_STR" >> $GITHUB_ENV
+
+    - name: Upload Backup to Azure Storage
+      shell: bash
+      run: |
+        az storage blob upload --account-name ${{ env.storage_account }} --container-name database-backup \
+        --file ${BACKUP_FILE_NAME}.tar.gz --name ${BACKUP_FILE_NAME}.tar.gz --overwrite \
+        --connection-string '${{ env.STORAGE_CONN_STR }}'
+        rm ${BACKUP_FILE_NAME}.tar.gz
+
+    - name: Disk cleanup
+      shell: bash
+      run: |
+        sudo rm -rf /usr/local/lib/android || true
+        sudo rm -rf /usr/share/dotnet || true
+        sudo rm -rf /opt/ghc || true
+
+    - name: Remove backup file
+      shell: bash
+      run: |
+        rm ${{ env.BACKUP_FILE_NAME }}.sql
+
+    - name: Check for Failure
+      if: ${{ failure() }}
+      uses: rtCamp/action-slack-notify@master
+      env:
+        SLACK_USERNAME: CI Deployment
+        SLACK_TITLE: Database backup failure
+        SLACK_MESSAGE: ${{ inputs.environment }} database backup job failed
+        SLACK_WEBHOOK: ${{ steps.slack-web-hook.outputs.SLACK_WEBHOOK }}
+        SLACK_COLOR: failure
+        SLACK_FOOTER: Sent from backup job in database-backup workflow
diff --git a/.github/workflows/aks-db-backup.yml b/.github/workflows/aks-db-backup.yml
@@ -0,0 +1,36 @@
+name: Backup AKS Database
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment
+        type: choice
+        options:
+          - development
+          - test
+          - preproduction
+          - production
+  schedule: # 01:00 UTC
+    - cron: "0 1 * * *"
+
+jobs:
+  backup:
+    name: Backup AKS Database
+    runs-on: ubuntu-latest
+    strategy:
+        max-parallel: 1
+        matrix:
+          environment: [development, test, preproduction, production]
+    environment:
+         name: ${{matrix.environment}}_aks
+    concurrency: ${{matrix.environment}}_${{github.event.number}}_aks
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - uses: ./.github/workflows/actions/database-backup
+        id: aks_db_backup
+        with:
+          azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
+          environment: ${{ matrix.environment }}
+
diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml
@@ -140,3 +140,4 @@ jobs:
           docker_image: ${{ needs.docker.outputs.docker_image }}
           azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
           arm-access-key: ${{ secrets.ARM_ACCESS_KEY }}
+
diff --git a/.github/workflows/database-backup.yml b/.github/workflows/database-backup.yml
@@ -2,6 +2,12 @@ name: Backup Database to Azure Storage
 
 on:
   workflow_dispatch:
+    inputs:
+      environment:
+        type: choice
+        options:
+          - dev
+          - test
   schedule: # 01:00 UTC
     - cron: "0 1 * * *"
 
@@ -23,6 +29,7 @@ jobs:
         shell: bash
         run: |
           tf_vars_file=terraform/paas/workspace_variables/production.tfvars.json
+          echo "namespace=$(jq -r '.namespace' ${tf_vars_file})" >> $GITHUB_ENV
           echo "KEY_VAULT_NAME=$(jq -r '.key_vault_name' ${tf_vars_file})" >> $GITHUB_ENV
           echo "PAAS_SPACE=$(jq -r '.paas_space' ${tf_vars_file})" >> $GITHUB_ENV
 

diff --git a/terraform/aks/databases.tf b/terraform/aks/databases.tf
@@ -14,6 +14,8 @@ module "postgres" {
   azure_enable_monitoring   = var.enable_monitoring
   azure_extensions          = ["plpgsql", "citext", "uuid-ossp"]
   server_version            = "14"
+
+  azure_enable_backup_storage = var.azure_enable_backup_storage
 }
 
 module "redis" {

diff --git a/terraform/aks/variables.tf b/terraform/aks/variables.tf
@@ -113,6 +113,10 @@ variable "inf_vault_name" {
   description = "infrastructure kv name"
 }
 
+variable "azure_enable_backup_storage" {
+  default = false  
+}
+
 variable "review_url_db_name" {
   default     = null
   description = "the name of the secret storing review db url"

diff --git a/terraform/aks/workspace_variables/production_aks.tfvars.json b/terraform/aks/workspace_variables/production_aks.tfvars.json
@@ -22,5 +22,6 @@
   "inf_vault_name": "s189p01-faltrn-pd-inf-kv",
   "key_vault_resource_group": "s189p01-faltrn-pd-rg",
   "worker_replicas": 2,
-  "replicas": 2
+  "replicas": 2,
+  "azure_enable_backup_storage": true
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -140,3 +140,4 @@ jobs:
		docker_image: ${{ needs.docker.outputs.docker_image }}
		azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
		arm-access-key: ${{ secrets.ARM_ACCESS_KEY }}