From 7cfc6aeca08a21e1b9396a1c32967a7b5cda2244 Mon Sep 17 00:00:00 2001
From: Andrew Moore <andrew.moore@digital.justice.gov.uk>
Date: Mon, 18 Nov 2024 14:18:31 +0000
Subject: [PATCH 1/3] feat: add opensearch-backup job

---
 jobs/opensearch-backup/Chart.yaml             |  5 ++
 .../templates/configmap.yaml                  | 67 +++++++++++++++++++
 jobs/opensearch-backup/templates/job.yaml     | 49 ++++++++++++++
 jobs/opensearch-backup/values.yaml            | 13 ++++
 4 files changed, 134 insertions(+)
 create mode 100644 jobs/opensearch-backup/Chart.yaml
 create mode 100644 jobs/opensearch-backup/templates/configmap.yaml
 create mode 100644 jobs/opensearch-backup/templates/job.yaml
 create mode 100644 jobs/opensearch-backup/values.yaml

diff --git a/jobs/opensearch-backup/Chart.yaml b/jobs/opensearch-backup/Chart.yaml
new file mode 100644
index 0000000..2d31648
--- /dev/null
+++ b/jobs/opensearch-backup/Chart.yaml
@@ -0,0 +1,5 @@
+apiVersion: v2
+name: delius-alfresco-opensearch-backup
+version: 0.0.1
+description: A Helm chart for backing up AWS OpenSearch indices
+type: application
diff --git a/jobs/opensearch-backup/templates/configmap.yaml b/jobs/opensearch-backup/templates/configmap.yaml
new file mode 100644
index 0000000..5cf804b
--- /dev/null
+++ b/jobs/opensearch-backup/templates/configmap.yaml
@@ -0,0 +1,67 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ include "opensearch-backup.fullname" . }}-script'
+data:
+  backup.sh: |
+    #!/bin/sh
+    set -e
+    echo "Starting OpenSearch backup process..."
+    
+    # Set timestamp for snapshot name
+    TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+    SNAPSHOT_NAME="${ENVIRONMENT}-${SNAPSHOT_PREFIX}-${TIMESTAMP}"
+    
+    echo "Creating snapshot: ${SNAPSHOT_NAME}"
+    echo "Repository: ${SNAPSHOT_REPOSITORY}"
+    echo "Indices to backup: ${INDICES}"
+    
+    # Check if repository exists
+    REPO_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY")
+    
+    if [ "$REPO_CHECK" = "404" ]; then
+      echo "Repository does not exist. Creating snapshot repository..."
+      curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{
+        \"type\": \"s3\",
+        \"settings\": {
+          \"bucket\": \"$S3_BUCKET_NAME\",
+          \"region\": \"$REGION\",
+          \"role_arn\": \"$SNAPSHOT_ROLE_ARN\"
+        }
+      }"
+      # Wait a moment for repository creation
+      sleep 5
+    else
+      echo "Repository already exists. Proceeding with backup..."
+    fi
+    
+    # Create the snapshot
+    echo "Creating snapshot..."
+    curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME" -H 'Content-Type: application/json' -d "{
+        \"indices\": \"$INDICES\",
+        \"include_global_state\": false
+    }"
+    
+    # Monitor snapshot progress
+    echo "Monitoring snapshot progress..."
+    while true; do
+        STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_status" | grep -o '"state":"[^"]*"' | cut -d'"' -f4)
+    
+        if [ "$STATUS" = "SUCCESS" ]; then
+        echo "Snapshot completed successfully"
+        break
+        elif [ "$STATUS" = "FAILED" ]; then
+        echo "Snapshot failed"
+        exit 1
+        else
+        echo "Snapshot in progress... (Status: $STATUS)"
+        sleep 10
+        fi
+    done
+    
+    # List snapshots to verify
+    echo "Listing snapshots in repository..."
+    curl -XGET "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/_all"
+    
+    echo "Backup process completed successfully"
diff --git a/jobs/opensearch-backup/templates/job.yaml b/jobs/opensearch-backup/templates/job.yaml
new file mode 100644
index 0000000..4162d87
--- /dev/null
+++ b/jobs/opensearch-backup/templates/job.yaml
@@ -0,0 +1,49 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "opensearch-backup.fullname" . }}
+spec:
+  template:
+    spec:
+      serviceAccountName: "{{ $.Values.opensearch.serviceAccountPrefix }}-{{ $.Values.opensearch.environment }}"
+      containers: opensearch-backup
+      image: alpine/curl:latest
+      command:
+        - /bin/sh
+        - /scripts/backup.sh
+      env:
+        - name: OPENSEARCH_ENDPOINT
+          valueFrom:
+            secretKeyRef:
+              name: {{ $.Values.opensearch.endpointSecretName }}
+              key: {{ $.Values.opensearch.endpointSecretKey }}
+        - name: SNAPSHOT_ROLE_ARN
+          valueFrom:
+            secretKeyRef:
+              name: {{ $.Values.opensearch.endpointSecretName }}
+              key: {{ $.Values.opensearch.snapshotRoleArnKey }}
+        - name: S3_BUCKET_NAME
+          valueFrom:
+            secretKeyRef:
+              name: {{ $.Values.opensearch.s3BucketSecretName }}
+              key: {{ $.Values.opensearch.s3BucketNameKey }}
+        - name: SNAPSHOT_REPOSITORY
+          value: {{ $.Values.opensearch.repository }}
+        - name: INDICES
+          value: {{ $.Values.opensearch.indices }}
+        - name: SNAPSHOT_PREFIX
+          value: {{ $.Values.opensearch.snapshotPrefix }}
+        - name: ENVIRONMENT
+          value: {{ $.Values.opensearch.environment }}
+        - name: REGION
+          value: {{ $.Values.opensearch.region }}
+      volumeMounts:
+        - name: script
+          mountPath: /scripts
+      volumes:
+        - name: script
+          configMap:
+            name: '{{ include "opensearch-backup.fullname" . }}-script'
+            defaultMode: 0755
+      restartPolicy: Never
diff --git a/jobs/opensearch-backup/values.yaml b/jobs/opensearch-backup/values.yaml
new file mode 100644
index 0000000..2616451
--- /dev/null
+++ b/jobs/opensearch-backup/values.yaml
@@ -0,0 +1,13 @@
+---
+opensearch:
+  repository: "daily-backups"
+  indices: "*" # Backup all indices by default
+  snapshotPrefix: "backup"
+  serviceAccountPrefix: "hmpps-migration"
+  endpointSecretName: "opensearch-output"
+  endpointSecretKey: "PROXY_URL"
+  snapshotRoleArnKey: "SNAPSHOT_ROLE_ARN"
+  s3BucketSecretName: "s3-opensearch-snapshots-bucket-output"
+  s3BucketNameKey: "BUCKET_NAME"
+  region: "eu-west-2"
+  envrionment: "" # Default empty, will be set by GitHub Action

From 031a908fd7ea5ad6c667c3eefd8d29b6d749bb51 Mon Sep 17 00:00:00 2001
From: Andrew Moore <andrew.moore@digital.justice.gov.uk>
Date: Mon, 18 Nov 2024 16:14:24 +0000
Subject: [PATCH 2/3] feat: add opensearch-backup gha workflow

---
 .github/workflows/opensearch-backup.yml       |  61 +++++++++++
 .../templates/configmap.yaml                  | 101 ++++++++++++------
 jobs/opensearch-backup/templates/job.yaml     |  81 +++++++-------
 jobs/opensearch-backup/values.yaml            |   4 +-
 4 files changed, 174 insertions(+), 73 deletions(-)
 create mode 100644 .github/workflows/opensearch-backup.yml

diff --git a/.github/workflows/opensearch-backup.yml b/.github/workflows/opensearch-backup.yml
new file mode 100644
index 0000000..e6f1683
--- /dev/null
+++ b/.github/workflows/opensearch-backup.yml
@@ -0,0 +1,61 @@
+---
+name: OpenSearch Backup
+
+on:
+  schedule:
+    - cron: '0 1 * * *'  # Run at 1am UTC daily
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Environment to backup'
+        required: true
+        type: choice
+        options:
+          - poc
+          - dev
+          - test
+          - stage
+          - preprod
+          
+jobs:
+  backup:
+    name: Backup OpenSearch
+    runs-on: ubuntu-latest
+    
+    strategy:
+      matrix:
+        environment: 
+          - poc
+          - dev
+          - test
+          - stage
+          # - preprod
+        exclude:
+          - ${{ github.event_name == 'workflow_dispatch' && environment != github.event.inputs.environment }}
+
+    environment: ${{ matrix.environment || github.event.inputs.environment }}-preapproved
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.2.2
+      - name: Configure kubectl
+        run: |
+          echo "${{ secrets.KUBE_CERT }}" > ca.crt
+          kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER}
+          kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }}
+          kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=${KUBE_NAMESPACE}
+          kubectl config use-context ${KUBE_CLUSTER}
+        env:
+          KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }}
+          KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }}
+      - name: Backup OpenSearch
+        working-directory: jobs/opensearch-backup
+        run: |
+          set -xeo pipefail
+          
+          helm install opensearch-backup . \
+            --set opensearch.environment=${{ matrix.environment || github.event.inputs.environment }}
+          
+          kubectl wait jobs -l name-prefix=opensearch-backup --for=condition=complete --timeout=1h
+      - name: Cleanup
+        run: helm uninstall opensearch-backup --ignore-not-found
diff --git a/jobs/opensearch-backup/templates/configmap.yaml b/jobs/opensearch-backup/templates/configmap.yaml
index 5cf804b..0cb8273 100644
--- a/jobs/opensearch-backup/templates/configmap.yaml
+++ b/jobs/opensearch-backup/templates/configmap.yaml
@@ -2,66 +2,99 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: '{{ include "opensearch-backup.fullname" . }}-script'
+  name: opensearch-backup-script
 data:
   backup.sh: |
     #!/bin/sh
     set -e
+
     echo "Starting OpenSearch backup process..."
-    
+
+    # Validate required environment variables
+    if [ -z "$OPENSEARCH_ENDPOINT" ] || [ -z "$S3_BUCKET_NAME" ] || [ -z "$SNAPSHOT_REPOSITORY" ] || [ -z "$INDICES" ] || [ -z "$REGION" ]; then
+      echo "Error: Required environment variables are not set"
+      echo "Required variables: OPENSEARCH_ENDPOINT, S3_BUCKET_NAME, SNAPSHOT_REPOSITORY, INDICES, REGION"
+      exit 1
+    fi
+
     # Set timestamp for snapshot name
     TIMESTAMP=$(date +%Y%m%d-%H%M%S)
-    SNAPSHOT_NAME="${ENVIRONMENT}-${SNAPSHOT_PREFIX}-${TIMESTAMP}"
-    
-    echo "Creating snapshot: ${SNAPSHOT_NAME}"
-    echo "Repository: ${SNAPSHOT_REPOSITORY}"
-    echo "Indices to backup: ${INDICES}"
-    
+    SNAPSHOT_NAME="${SNAPSHOT_PREFIX:-backup}-${ENVIRONMENT:-default}-${TIMESTAMP}"
+
+    echo "Creating snapshot: $SNAPSHOT_NAME"
+    echo "Repository: $SNAPSHOT_REPOSITORY"
+    echo "Indices to backup: $INDICES"
+    echo "---"
+
     # Check if repository exists
     REPO_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY")
-    
+
     if [ "$REPO_CHECK" = "404" ]; then
       echo "Repository does not exist. Creating snapshot repository..."
-      curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{
+      RESPONSE=$(curl -s -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{
         \"type\": \"s3\",
         \"settings\": {
           \"bucket\": \"$S3_BUCKET_NAME\",
           \"region\": \"$REGION\",
           \"role_arn\": \"$SNAPSHOT_ROLE_ARN\"
         }
-      }"
-      # Wait a moment for repository creation
-      sleep 5
+      }")
+      if echo "$RESPONSE" | grep -q '"acknowledged":true'; then
+        echo "Repository created successfully"
+      else
+        echo "Failed to create repository: $RESPONSE"
+        exit 1
+      fi
+      echo "---"
     else
-      echo "Repository already exists. Proceeding with backup..."
+      echo "Repository already exists"
+      echo "---"
     fi
-    
+
     # Create the snapshot
     echo "Creating snapshot..."
-    curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME" -H 'Content-Type: application/json' -d "{
-        \"indices\": \"$INDICES\",
-        \"include_global_state\": false
-    }"
-    
+    RESPONSE=$(curl -s -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME" -H 'Content-Type: application/json' -d "{
+      \"indices\": \"$INDICES\",
+      \"include_global_state\": false
+    }")
+
+    if ! echo "$RESPONSE" | grep -q '"accepted":true'; then
+      echo "Failed to create snapshot: $RESPONSE"
+      exit 1
+    fi
+
     # Monitor snapshot progress
     echo "Monitoring snapshot progress..."
     while true; do
-        STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_status" | grep -o '"state":"[^"]*"' | cut -d'"' -f4)
-    
-        if [ "$STATUS" = "SUCCESS" ]; then
-        echo "Snapshot completed successfully"
+      CURRENT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
+      SNAPSHOT_STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_status")
+      STATE=$(echo "$SNAPSHOT_STATUS" | grep -o '"state":"[^"]*"' | cut -d'"' -f4)
+      
+      if [ "$STATE" = "SUCCESS" ]; then
+        echo "[$CURRENT_TIME] Snapshot completed successfully"
         break
-        elif [ "$STATUS" = "FAILED" ]; then
-        echo "Snapshot failed"
+      elif [ "$STATE" = "FAILED" ]; then
+        echo "[$CURRENT_TIME] Snapshot failed"
         exit 1
-        else
-        echo "Snapshot in progress... (Status: $STATUS)"
+      else
+        echo "[$CURRENT_TIME] Snapshot in progress... (Status: $STATE)"
         sleep 10
-        fi
+      fi
     done
-    
-    # List snapshots to verify
-    echo "Listing snapshots in repository..."
-    curl -XGET "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/_all"
-    
+    echo "---"
+
+    # Get final snapshot details
+    echo "Snapshot Details:"
+    SNAPSHOT_INFO=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME")
+
+    # Parse and display relevant information
+    INDICES_COUNT=$(echo "$SNAPSHOT_INFO" | grep -o '"indices":\[[^]]*\]' | grep -o ',' | wc -l)
+    INDICES_COUNT=$((INDICES_COUNT + 1))
+    START_TIME=$(echo "$SNAPSHOT_INFO" | grep -o '"start_time":"[^"]*"' | cut -d'"' -f4)
+
+    echo "- Name: $SNAPSHOT_NAME"
+    echo "- Start Time: $START_TIME"
+    echo "- Indices Backed Up: $INDICES_COUNT"
+    echo "---"
+
     echo "Backup process completed successfully"
diff --git a/jobs/opensearch-backup/templates/job.yaml b/jobs/opensearch-backup/templates/job.yaml
index 4162d87..64013bd 100644
--- a/jobs/opensearch-backup/templates/job.yaml
+++ b/jobs/opensearch-backup/templates/job.yaml
@@ -2,48 +2,55 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ include "opensearch-backup.fullname" . }}
+  name: opensearch-backup
 spec:
   template:
     spec:
       serviceAccountName: "{{ $.Values.opensearch.serviceAccountPrefix }}-{{ $.Values.opensearch.environment }}"
-      containers: opensearch-backup
-      image: alpine/curl:latest
-      command:
-        - /bin/sh
-        - /scripts/backup.sh
-      env:
-        - name: OPENSEARCH_ENDPOINT
-          valueFrom:
-            secretKeyRef:
-              name: {{ $.Values.opensearch.endpointSecretName }}
-              key: {{ $.Values.opensearch.endpointSecretKey }}
-        - name: SNAPSHOT_ROLE_ARN
-          valueFrom:
-            secretKeyRef:
-              name: {{ $.Values.opensearch.endpointSecretName }}
-              key: {{ $.Values.opensearch.snapshotRoleArnKey }}
-        - name: S3_BUCKET_NAME
-          valueFrom:
-            secretKeyRef:
-              name: {{ $.Values.opensearch.s3BucketSecretName }}
-              key: {{ $.Values.opensearch.s3BucketNameKey }}
-        - name: SNAPSHOT_REPOSITORY
-          value: {{ $.Values.opensearch.repository }}
-        - name: INDICES
-          value: {{ $.Values.opensearch.indices }}
-        - name: SNAPSHOT_PREFIX
-          value: {{ $.Values.opensearch.snapshotPrefix }}
-        - name: ENVIRONMENT
-          value: {{ $.Values.opensearch.environment }}
-        - name: REGION
-          value: {{ $.Values.opensearch.region }}
-      volumeMounts:
-        - name: script
-          mountPath: /scripts
+      securityContext:
+        allowPrivilegeEscalation: false
+        privileged: false
+        readOnlyRootFilesystem: false
+        runAsNonRoot: true
+        runAsUser: 999
+      containers:
+        - name: opensearch-backup
+          image: ghcr.io/ministryofjustice/hmpps-delius-alfresco-utils:latest
+          command:
+            - /bin/sh
+            - /scripts/backup.sh
+          env:
+            - name: OPENSEARCH_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.opensearch.endpointSecretName }}
+                  key: {{ $.Values.opensearch.endpointSecretKey }}
+            - name: S3_BUCKET_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.opensearch.s3BucketSecretName }}
+                  key: {{ $.Values.opensearch.s3BucketNameKey }}
+            - name: SNAPSHOT_ROLE_ARN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.opensearch.endpointSecretName }}
+                  key: {{ $.Values.opensearch.snapshotRoleArnKey }}
+            - name: SNAPSHOT_REPOSITORY
+              value: "{{ $.Values.opensearch.repository }}"
+            - name: INDICES
+              value: "{{ $.Values.opensearch.indices }}"
+            - name: SNAPSHOT_PREFIX
+              value: "{{ $.Values.opensearch.snapshotPrefix }}"
+            - name: ENVIRONMENT
+              value: "{{ $.Values.opensearch.environment }}"
+            - name: REGION
+              value: "{{ $.Values.opensearch.region }}"
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
       volumes:
         - name: script
           configMap:
-            name: '{{ include "opensearch-backup.fullname" . }}-script'
+            name: opensearch-backup-script
             defaultMode: 0755
-      restartPolicy: Never
+      restartPolicy: Never
\ No newline at end of file
diff --git a/jobs/opensearch-backup/values.yaml b/jobs/opensearch-backup/values.yaml
index 2616451..18ad57a 100644
--- a/jobs/opensearch-backup/values.yaml
+++ b/jobs/opensearch-backup/values.yaml
@@ -1,7 +1,7 @@
 ---
 opensearch:
   repository: "daily-backups"
-  indices: "*" # Backup all indices by default
+  indices: "alfresco"
   snapshotPrefix: "backup"
   serviceAccountPrefix: "hmpps-migration"
   endpointSecretName: "opensearch-output"
@@ -10,4 +10,4 @@ opensearch:
   s3BucketSecretName: "s3-opensearch-snapshots-bucket-output"
   s3BucketNameKey: "BUCKET_NAME"
   region: "eu-west-2"
-  envrionment: "" # Default empty, will be set by GitHub Action
+  environment: "" # Default empty, will be set by GitHub Action

From 4bee540d9acd39c7805a5a22620c60f2ac902195 Mon Sep 17 00:00:00 2001
From: Andrew Moore <andrew.moore@digital.justice.gov.uk>
Date: Mon, 18 Nov 2024 16:14:44 +0000
Subject: [PATCH 3/3] feat: tidy up opensearch-backup job and script

---
 jobs/opensearch-backup/templates/job.yaml | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/jobs/opensearch-backup/templates/job.yaml b/jobs/opensearch-backup/templates/job.yaml
index 64013bd..bcb7bad 100644
--- a/jobs/opensearch-backup/templates/job.yaml
+++ b/jobs/opensearch-backup/templates/job.yaml
@@ -7,12 +7,6 @@ spec:
   template:
     spec:
       serviceAccountName: "{{ $.Values.opensearch.serviceAccountPrefix }}-{{ $.Values.opensearch.environment }}"
-      securityContext:
-        allowPrivilegeEscalation: false
-        privileged: false
-        readOnlyRootFilesystem: false
-        runAsNonRoot: true
-        runAsUser: 999
       containers:
         - name: opensearch-backup
           image: ghcr.io/ministryofjustice/hmpps-delius-alfresco-utils:latest
@@ -48,9 +42,20 @@ spec:
           volumeMounts:
             - name: script
               mountPath: /scripts
+          securityContext:
+            allowPrivilegeEscalation: false
+            privileged: false
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 999
+            capabilities:
+              drop:
+                - ALL
+            seccompProfile:
+              type: RuntimeDefault
       volumes:
         - name: script
           configMap:
             name: opensearch-backup-script
             defaultMode: 0755
-      restartPolicy: Never
\ No newline at end of file
+      restartPolicy: Never