feat: add opensearch-backup gha workflow

ministryofjustice · Nov 18, 2024 · 031a908 · 031a908
1 parent 7cfc6ae
commit 031a908
Show file tree

Hide file tree

Showing 4 changed files with 174 additions and 73 deletions.
diff --git a/.github/workflows/opensearch-backup.yml b/.github/workflows/opensearch-backup.yml
@@ -0,0 +1,61 @@
+---
+name: OpenSearch Backup
+
+on:
+  schedule:
+    - cron: '0 1 * * *'  # Run at 1am UTC daily
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Environment to backup'
+        required: true
+        type: choice
+        options:
+          - poc
+          - dev
+          - test
+          - stage
+          - preprod
+
+jobs:
+  backup:
+    name: Backup OpenSearch
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        environment: 
+          - poc
+          - dev
+          - test
+          - stage
+          # - preprod
+        exclude:
+          - ${{ github.event_name == 'workflow_dispatch' && environment != github.event.inputs.environment }}
+
+    environment: ${{ matrix.environment || github.event.inputs.environment }}-preapproved
+
+    steps:
+      - name: Checkout code
+        uses: actions/[email protected]
+      - name: Configure kubectl
+        run: |
+          echo "${{ secrets.KUBE_CERT }}" > ca.crt
+          kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER}
+          kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }}
+          kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=${KUBE_NAMESPACE}
+          kubectl config use-context ${KUBE_CLUSTER}
+        env:
+          KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }}
+          KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }}
+      - name: Backup OpenSearch
+        working-directory: jobs/opensearch-backup
+        run: |
+          set -xeo pipefail
+          
+          helm install opensearch-backup . \
+            --set opensearch.environment=${{ matrix.environment || github.event.inputs.environment }}
+          
+          kubectl wait jobs -l name-prefix=opensearch-backup --for=condition=complete --timeout=1h
+      - name: Cleanup
+        run: helm uninstall opensearch-backup --ignore-not-found
diff --git a/jobs/opensearch-backup/templates/configmap.yaml b/jobs/opensearch-backup/templates/configmap.yaml
@@ -2,66 +2,99 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: '{{ include "opensearch-backup.fullname" . }}-script'
+  name: opensearch-backup-script
 data:
   backup.sh: |
     #!/bin/sh
     set -e
+
     echo "Starting OpenSearch backup process..."
-    
+
+    # Validate required environment variables
+    if [ -z "$OPENSEARCH_ENDPOINT" ] || [ -z "$S3_BUCKET_NAME" ] || [ -z "$SNAPSHOT_REPOSITORY" ] || [ -z "$INDICES" ] || [ -z "$REGION" ]; then
+      echo "Error: Required environment variables are not set"
+      echo "Required variables: OPENSEARCH_ENDPOINT, S3_BUCKET_NAME, SNAPSHOT_REPOSITORY, INDICES, REGION"
+      exit 1
+    fi
+
     # Set timestamp for snapshot name
     TIMESTAMP=$(date +%Y%m%d-%H%M%S)
-    SNAPSHOT_NAME="${ENVIRONMENT}-${SNAPSHOT_PREFIX}-${TIMESTAMP}"
-    
-    echo "Creating snapshot: ${SNAPSHOT_NAME}"
-    echo "Repository: ${SNAPSHOT_REPOSITORY}"
-    echo "Indices to backup: ${INDICES}"
-    
+    SNAPSHOT_NAME="${SNAPSHOT_PREFIX:-backup}-${ENVIRONMENT:-default}-${TIMESTAMP}"
+
+    echo "Creating snapshot: $SNAPSHOT_NAME"
+    echo "Repository: $SNAPSHOT_REPOSITORY"
+    echo "Indices to backup: $INDICES"
+    echo "---"
+
     # Check if repository exists
     REPO_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY")
-    
+
     if [ "$REPO_CHECK" = "404" ]; then
       echo "Repository does not exist. Creating snapshot repository..."
-      curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{
+      RESPONSE=$(curl -s -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{
         \"type\": \"s3\",
         \"settings\": {
           \"bucket\": \"$S3_BUCKET_NAME\",
           \"region\": \"$REGION\",
           \"role_arn\": \"$SNAPSHOT_ROLE_ARN\"
         }
-      }"
-      # Wait a moment for repository creation
-      sleep 5
+      }")
+      if echo "$RESPONSE" | grep -q '"acknowledged":true'; then
+        echo "Repository created successfully"
+      else
+        echo "Failed to create repository: $RESPONSE"
+        exit 1
+      fi
+      echo "---"
     else
-      echo "Repository already exists. Proceeding with backup..."
+      echo "Repository already exists"
+      echo "---"
     fi
-    
+
     # Create the snapshot
     echo "Creating snapshot..."
-    curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME" -H 'Content-Type: application/json' -d "{
-        \"indices\": \"$INDICES\",
-        \"include_global_state\": false
-    }"
-    
+    RESPONSE=$(curl -s -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME" -H 'Content-Type: application/json' -d "{
+      \"indices\": \"$INDICES\",
+      \"include_global_state\": false
+    }")
+
+    if ! echo "$RESPONSE" | grep -q '"accepted":true'; then
+      echo "Failed to create snapshot: $RESPONSE"
+      exit 1
+    fi
+
     # Monitor snapshot progress
     echo "Monitoring snapshot progress..."
     while true; do
-        STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_status" | grep -o '"state":"[^"]*"' | cut -d'"' -f4)
-    
-        if [ "$STATUS" = "SUCCESS" ]; then
-        echo "Snapshot completed successfully"
+      CURRENT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
+      SNAPSHOT_STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_status")
+      STATE=$(echo "$SNAPSHOT_STATUS" | grep -o '"state":"[^"]*"' | cut -d'"' -f4)
+      
+      if [ "$STATE" = "SUCCESS" ]; then
+        echo "[$CURRENT_TIME] Snapshot completed successfully"
         break
-        elif [ "$STATUS" = "FAILED" ]; then
-        echo "Snapshot failed"
+      elif [ "$STATE" = "FAILED" ]; then
+        echo "[$CURRENT_TIME] Snapshot failed"
         exit 1
-        else
-        echo "Snapshot in progress... (Status: $STATUS)"
+      else
+        echo "[$CURRENT_TIME] Snapshot in progress... (Status: $STATE)"
         sleep 10
-        fi
+      fi
     done
-    
-    # List snapshots to verify
-    echo "Listing snapshots in repository..."
-    curl -XGET "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/_all"
-    
+    echo "---"
+
+    # Get final snapshot details
+    echo "Snapshot Details:"
+    SNAPSHOT_INFO=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME")
+
+    # Parse and display relevant information
+    INDICES_COUNT=$(echo "$SNAPSHOT_INFO" | grep -o '"indices":\[[^]]*\]' | grep -o ',' | wc -l)
+    INDICES_COUNT=$((INDICES_COUNT + 1))
+    START_TIME=$(echo "$SNAPSHOT_INFO" | grep -o '"start_time":"[^"]*"' | cut -d'"' -f4)
+
+    echo "- Name: $SNAPSHOT_NAME"
+    echo "- Start Time: $START_TIME"
+    echo "- Indices Backed Up: $INDICES_COUNT"
+    echo "---"
+
     echo "Backup process completed successfully"
diff --git a/jobs/opensearch-backup/templates/job.yaml b/jobs/opensearch-backup/templates/job.yaml
@@ -2,48 +2,55 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ include "opensearch-backup.fullname" . }}
+  name: opensearch-backup
 spec:
   template:
     spec:
       serviceAccountName: "{{ $.Values.opensearch.serviceAccountPrefix }}-{{ $.Values.opensearch.environment }}"
-      containers: opensearch-backup
-      image: alpine/curl:latest
-      command:
-        - /bin/sh
-        - /scripts/backup.sh
-      env:
-        - name: OPENSEARCH_ENDPOINT
-          valueFrom:
-            secretKeyRef:
-              name: {{ $.Values.opensearch.endpointSecretName }}
-              key: {{ $.Values.opensearch.endpointSecretKey }}
-        - name: SNAPSHOT_ROLE_ARN
-          valueFrom:
-            secretKeyRef:
-              name: {{ $.Values.opensearch.endpointSecretName }}
-              key: {{ $.Values.opensearch.snapshotRoleArnKey }}
-        - name: S3_BUCKET_NAME
-          valueFrom:
-            secretKeyRef:
-              name: {{ $.Values.opensearch.s3BucketSecretName }}
-              key: {{ $.Values.opensearch.s3BucketNameKey }}
-        - name: SNAPSHOT_REPOSITORY
-          value: {{ $.Values.opensearch.repository }}
-        - name: INDICES
-          value: {{ $.Values.opensearch.indices }}
-        - name: SNAPSHOT_PREFIX
-          value: {{ $.Values.opensearch.snapshotPrefix }}
-        - name: ENVIRONMENT
-          value: {{ $.Values.opensearch.environment }}
-        - name: REGION
-          value: {{ $.Values.opensearch.region }}
-      volumeMounts:
-        - name: script
-          mountPath: /scripts
+      securityContext:
+        allowPrivilegeEscalation: false
+        privileged: false
+        readOnlyRootFilesystem: false
+        runAsNonRoot: true
+        runAsUser: 999
+      containers:
+        - name: opensearch-backup
+          image: ghcr.io/ministryofjustice/hmpps-delius-alfresco-utils:latest
+          command:
+            - /bin/sh
+            - /scripts/backup.sh
+          env:
+            - name: OPENSEARCH_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.opensearch.endpointSecretName }}
+                  key: {{ $.Values.opensearch.endpointSecretKey }}
+            - name: S3_BUCKET_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.opensearch.s3BucketSecretName }}
+                  key: {{ $.Values.opensearch.s3BucketNameKey }}
+            - name: SNAPSHOT_ROLE_ARN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.opensearch.endpointSecretName }}
+                  key: {{ $.Values.opensearch.snapshotRoleArnKey }}
+            - name: SNAPSHOT_REPOSITORY
+              value: "{{ $.Values.opensearch.repository }}"
+            - name: INDICES
+              value: "{{ $.Values.opensearch.indices }}"
+            - name: SNAPSHOT_PREFIX
+              value: "{{ $.Values.opensearch.snapshotPrefix }}"
+            - name: ENVIRONMENT
+              value: "{{ $.Values.opensearch.environment }}"
+            - name: REGION
+              value: "{{ $.Values.opensearch.region }}"
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
       volumes:
         - name: script
           configMap:
-            name: '{{ include "opensearch-backup.fullname" . }}-script'
+            name: opensearch-backup-script
             defaultMode: 0755
-      restartPolicy: Never
+      restartPolicy: Never
diff --git a/jobs/opensearch-backup/values.yaml b/jobs/opensearch-backup/values.yaml
@@ -1,7 +1,7 @@
 ---
 opensearch:
   repository: "daily-backups"
-  indices: "*" # Backup all indices by default
+  indices: "alfresco"
   snapshotPrefix: "backup"
   serviceAccountPrefix: "hmpps-migration"
   endpointSecretName: "opensearch-output"
@@ -10,4 +10,4 @@ opensearch:
   s3BucketSecretName: "s3-opensearch-snapshots-bucket-output"
   s3BucketNameKey: "BUCKET_NAME"
   region: "eu-west-2"
-  envrionment: "" # Default empty, will be set by GitHub Action
+  environment: "" # Default empty, will be set by GitHub Action