feat: add opensearch-refresh helm chart and workflow (#116)

ministryofjustice · Dec 5, 2024 · b8ddccf · b8ddccf
1 parent efd6a64
commit b8ddccf
Show file tree

Hide file tree

Showing 5 changed files with 407 additions and 0 deletions.
diff --git a/.github/workflows/opensearch-refresh.yml b/.github/workflows/opensearch-refresh.yml
@@ -0,0 +1,99 @@
+name: OpenSearch Refresh
+
+on:
+  workflow_dispatch:
+    inputs:
+      sourceEnv:
+        description: 'Source environment'
+        required: true
+        type: choice
+        options:
+          - dev
+          - test
+          - stage
+          - preprod
+          - prod
+      destEnv:
+        description: 'Destination environment'
+        required: true
+        type: choice
+        options:
+          - dev
+          - test
+          - stage
+          - preprod
+          - prod
+
+jobs:
+  create-snapshot:
+    name: Create snapshot in the source environment
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ github.event.inputs.sourceEnv }}
+    outputs:
+      snapshotName: ${{ steps.get_snapshot_name.outputs.SNAPSHOT_NAME }}
+    steps:
+      - name: Checkout code
+        uses: actions/[email protected]
+      - name: Configure kubectl
+        run: |
+          echo "${{ secrets.KUBE_CERT }}" > ca.crt
+          kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER}
+          kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }}
+          kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=${KUBE_NAMESPACE}
+          kubectl config use-context ${KUBE_CLUSTER}
+        env:
+          KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }}
+          KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }}
+      - name: Create snapshot
+        working-directory: jobs/refresh-opensearch
+        run: |
+            helm install create-opensearch-refresh-snapshot ./refresh-opensearch
+            --set jobType=create
+            --set sourceEnv=${{ github.event.inputs.sourceEnv }}
+            --set destEnv=${{ github.event.inputs.destEnv }}
+      - name: Wait for create job
+        run: |
+          kubectl wait job/create-opensearch-refresh-snapshot --for=condition=complete --timeout=3h
+      - name: Get snapshot name
+        run: |
+          POD_NAME=$(kubectl get pods --selector=job-name=create-opensearch-refresh-snapshot -o jsonpath="{.items[0].metadata.name}")
+          SNAPSHOT_NAME=$(kubectl logs $POD_NAME | tail -n 1)
+          echo "SNAPSHOT_NAME=${SNAPSHOT_NAME}" >> $GITHUB_OUTPUT
+      - name: Cleanup
+        if: always()
+        run: helm uninstall create-opensearch-refresh-snapshot --ignore-not-found
+  restore-snapshot:
+    needs: create-snapshot
+    name: Restore snapshot in the destination environment
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ github.event.inputs.destEnv }}
+    steps:
+      - name: Checkout code
+        uses: actions/[email protected]
+      - name: Configure kubectl
+        run: |
+          echo "${{ secrets.KUBE_CERT }}" > ca.crt
+          kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER}
+          kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }}
+          kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=${KUBE_NAMESPACE}
+          kubectl config use-context ${KUBE_CLUSTER}
+        env:
+          KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }}
+          KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }}
+      - name: Restore snapshot
+        working-directory: jobs/refresh-opensearch
+        run: |
+            helm install restore-opensearch-refresh-snapshot ./refresh-opensearch
+            --set jobType=restore
+            --set sourceEnv=${{ github.event.inputs.sourceEnv }}
+            --set destEnv=${{ github.event.inputs.destEnv }}
+            --set snapshotName=${{ needs.create-snapshot.outputs.snapshotName }}
+      - name: Wait for restore job
+        run: |
+          kubectl wait job/restore-opensearch-refresh-snapshot --for=condition=complete --timeout=3h
+      - name: Cleanup
+        if: always()
+        run: helm uninstall restore-opensearch-refresh-snapshot --ignore-not-found
+
diff --git a/jobs/refresh-opensearch/Chart.yaml b/jobs/refresh-opensearch/Chart.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: v2
+name: delius-alfresco-refresh-opensearch
+version: 0.0.1
+description: Helm chart for OpenSearch data migration between environments
+type: application
diff --git a/jobs/refresh-opensearch/templates/create-snapshot.yaml b/jobs/refresh-opensearch/templates/create-snapshot.yaml
@@ -0,0 +1,126 @@
+{{- if eq .Values.jobType "create" }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: create-opensearch-snapshot
+spec:
+  restartPolicy: Never
+  serviceAccountName: "{{ $.Values.job.serviceAccountPrefix }}-{{ $.Values.sourceEnv }}"
+  template:
+    spec:
+      containers:
+        - name: create-opensearch-snapshot
+          image: {{ .Values.job.image }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            privileged: false
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 999
+            capabilities:
+              drop:
+                - ALL
+            seccompProfile:
+            type: RuntimeDefault
+          env:
+            - name: OPENSEARCH_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.opensearch.endpointSecretName }}
+                  key: {{ $.Values.opensearch.endpointSecretKey }}
+            - name: S3_BUCKET_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.s3.s3BucketSecretName }}
+                  key: {{ $.Values.s3.s3BucketNameKey }}
+            - name: SNAPSHOT_ROLE_ARN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.opensearch.endpointSecretName }}
+                  key: {{ $.Values.opensearch.snapshotRoleArnKey }}
+            - name: SNAPSHOT_PREFIX
+              value: "{{ $.Values.opensearch.snapshotPrefix }}"
+            - name: SNAPSHOT_REPOSITORY
+              value: "{{ $.Values.opensearch.snapshotRepository }}"
+            - name: REGION
+              value: "{{ $.Values.s3.region }}"
+            - name: SOURCE_ENV
+              value: "{{ $.Values.sourceEnv }}"
+            - name: DEST_ENV
+              value: "{{ $.Values.destEnv }}"
+            - name: INDICES
+              value: "{{ $.Values.opensearch.indices }}"
+          command:
+          - /bin/sh
+          - -c
+          - |
+            TIMESTAMP=$(date "+%Y%m%d%H%M%S")
+            SNAPSHOT_NAME=${SNAPSHOT_PREFIX}-${SOURCE_ENV}-to-${DEST_ENV}-${TIMESTAMP}
+
+            echo "OPENSEARCH_ENDPOINT: ${OPENSEARCH_ENDPOINT}"
+            echo "S3_BUCKET_NAME: ${S3_BUCKET_NAME}"
+            echo "SNAPSHOT_ROLE_ARN: ${SNAPSHOT_ROLE_ARN}"
+            echo "SNAPSHOT_PREFIX: ${SNAPSHOT_PREFIX}"
+            echo "SNAPSHOT_REPOSITORY: ${SNAPSHOT_REPOSITORY}"
+            echo "REGION: ${REGION}"
+            echo "SOURCE_ENV: ${SOURCE_ENV}"
+            echo "DEST_ENV: ${DEST_ENV}"
+            echo "INDICES: ${INDICES}"
+
+            # Check if repository exists
+            REPO_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY")
+
+            if [ "$REPO_CHECK" = "404" ]; then
+              echo "Repository does not exist. Creating snapshot repository..."
+              RESPONSE=$(curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{
+                \"type\": \"s3\",
+                \"settings\": {
+                  \"bucket\": \"$S3_BUCKET_NAME\",
+                  \"region\": \"$REGION\",
+                  \"role_arn\": \"$SNAPSHOT_ROLE_ARN\"
+                }
+              }")
+              if echo "$RESPONSE" | grep -q '"acknowledged":true'; then
+                echo "Repository created successfully"
+              else
+                echo "Failed to create repository: $RESPONSE"
+                exit 1
+              fi
+            else
+              echo "Repository already exists"
+            fi
+
+            # Create snapshot
+            echo "Creating snapshot..."
+            RESPONSE=$(curl -s -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME" -H 'Content-Type: application/json' -d "{
+              \"indices\": \"$INDICES\",
+              \"include_global_state\": false
+            }")
+
+            if ! echo "$RESPONSE" | grep -q '"accepted":true'; then
+              echo "Failed to create snapshot: $RESPONSE"
+              exit 1
+            fi
+
+            # Monitor snapshot progress
+            echo "Monitoring snapshot progress..."
+            while true; do
+              CURRENT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
+              SNAPSHOT_STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_status")
+              STATE=$(echo "$SNAPSHOT_STATUS" | grep -o '"state":"[^"]*"' | cut -d'"' -f4)
+
+              if [ "$STATE" = "SUCCESS" ]; then
+                echo "[$CURRENT_TIME] Snapshot completed successfully"
+                break
+              elif [ "$STATE" = "FAILED" ]; then
+                echo "[$CURRENT_TIME] Snapshot failed"
+                exit 1
+              else
+                echo "[$CURRENT_TIME] Snapshot in progress... (Status: $STATE)"
+                sleep 10
+              fi
+            done
+
+            echo "${SNAPSHOT_NAME}"
+
+{{- end }}
diff --git a/jobs/refresh-opensearch/templates/restore-snapshot.yaml b/jobs/refresh-opensearch/templates/restore-snapshot.yaml
@@ -0,0 +1,139 @@
+{{- if eq .Values.jobType "restore" }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: restore-opensearch-snapshot
+spec:
+  restartPolicy: Never
+  serviceAccountName: "{{ $.Values.job.serviceAccountPrefix }}-{{ $.Values.destEnv }}"
+  template:
+    spec:
+      containers:
+        - name: restore-opensearch-snapshot
+          image: {{ .Values.job.image }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            privileged: false
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 999
+            capabilities:
+              drop:
+                - ALL
+            seccompProfile:
+            type: RuntimeDefault
+          env:
+            - name: OPENSEARCH_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.opensearch.endpointSecretName }}
+                  key: {{ $.Values.opensearch.endpointSecretKey }}
+            - name: SOURCE_S3_BUCKET_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ $.Values.s3.sourceS3BucketSecretName }}-{{ $.Values.sourceEnv }}"
+                  key: {{ $.Values.s3.s3BucketNameKey }}
+            - name: SNAPSHOT_ROLE_ARN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.opensearch.endpointSecretName }}
+                  key: {{ $.Values.opensearch.snapshotRoleArnKey }}
+            - name: SNAPSHOT_NAME
+              value: "{{ $.Values.opensearch.snapshotName }}"
+            - name: SNAPSHOT_REPOSITORY
+              value: "refresh-snapshots-{{ $.Values.sourceEnv }}"
+            - name: REGION
+              value: "{{ $.Values.s3.region }}"
+            - name: SOURCE_ENV
+              value: "{{ $.Values.sourceEnv }}"
+            - name: DEST_ENV
+              value: "{{ $.Values.destEnv }}"
+            - name: INDICES
+              value: "{{ $.Values.opensearch.indices }}"
+          command:
+            - /bin/sh
+            - -c
+            - |
+              echo "OPENSEARCH_ENDPOINT: ${OPENSEARCH_ENDPOINT}"
+              echo "SOURCE_S3_BUCKET_NAME: ${SOURCE_S3_BUCKET_NAME}"
+              echo "SNAPSHOT_ROLE_ARN: ${SNAPSHOT_ROLE_ARN}"
+              echo "SNAPSHOT_NAME: ${SNAPSHOT_NAME}"
+              echo "SNAPSHOT_REPOSITORY: ${SNAPSHOT_REPOSITORY}"
+              echo "REGION: ${REGION}"
+              echo "SOURCE_ENV: ${SOURCE_ENV}"
+              echo "DEST_ENV: ${DEST_ENV}"
+              echo "INDICES: ${INDICES}"
+
+              # Check if repository exists
+              REPO_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY")
+
+              if [ "$REPO_CHECK" = "404" ]; then
+                echo "Repository does not exist. Creating snapshot repository..."
+                RESPONSE=$(curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{
+                  \"type\": \"s3\",
+                  \"settings\": {
+                    \"bucket\": \"$SOURCE_S3_BUCKET_NAME\",
+                    \"region\": \"$REGION\",
+                    \"role_arn\": \"$SNAPSHOT_ROLE_ARN\"
+                  }
+                }")
+                if echo "$RESPONSE" | grep -q '"acknowledged":true'; then
+                  echo "Repository created successfully"
+                else
+                  echo "Failed to create repository: $RESPONSE"
+                  exit 1
+                fi
+              else
+                echo "Repository already exists"
+              fi
+
+              # Close indices before restore if they exist
+              echo "Checking and closing existing indices..."
+              for INDEX in $(echo "$INDICES" | tr ',' ' '); do
+                INDEX_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$OPENSEARCH_ENDPOINT/$INDEX")
+                if [ "$INDEX_CHECK" = "200" ]; then
+                  echo "Closing index $INDEX..."
+                  curl -XPOST "$OPENSEARCH_ENDPOINT/$INDEX/_close"
+                fi
+              done
+
+              # Restore snapshot
+              echo "Restoring snapshot..."
+              RESPONSE=$(curl -s -XPOST "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_restore" -H 'Content-Type: application/json' -d "{
+                \"indices\": \"$INDICES\",
+                \"include_global_state\": false
+              }")
+
+              if ! echo "$RESPONSE" | grep -q '"accepted":true'; then
+                echo "Failed to initiate restore: $RESPONSE"
+                exit 1
+              fi
+
+              # Monitor restore progress
+              echo "Monitoring restore progress..."
+              while true; do
+                CURRENT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
+
+                # Check recovery status
+                RECOVERY_STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_recovery" | grep -o '"stage":"[^"]*"' | sort -u)
+
+                if echo "$RECOVERY_STATUS" | grep -q "done"; then
+                  echo "[$CURRENT_TIME] Restore completed successfully"
+                  break
+                elif echo "$RECOVERY_STATUS" | grep -q "failed"; then
+                  echo "[$CURRENT_TIME] Restore failed"
+                  exit 1
+                else
+                  echo "[$CURRENT_TIME] Restore in progress... (Status: $RECOVERY_STATUS)"
+                  sleep 10
+                fi
+              done
+
+              # Open restored indices
+              echo "Opening restored indices..."
+              for INDEX in $(echo "$INDICES" | tr ',' ' '); do
+                echo "Opening index $INDEX..."
+                curl -XPOST "$OPENSEARCH_ENDPOINT/$INDEX/_open"
+              done
+
+  {{- end }}