diff --git a/.github/workflows/opensearch-backup.yml b/.github/workflows/opensearch-backup.yml new file mode 100644 index 0000000..e6f1683 --- /dev/null +++ b/.github/workflows/opensearch-backup.yml @@ -0,0 +1,61 @@ +--- +name: OpenSearch Backup + +on: + schedule: + - cron: '0 1 * * *' # Run at 1am UTC daily + workflow_dispatch: + inputs: + environment: + description: 'Environment to backup' + required: true + type: choice + options: + - poc + - dev + - test + - stage + - preprod + +jobs: + backup: + name: Backup OpenSearch + runs-on: ubuntu-latest + + strategy: + matrix: + environment: + - poc + - dev + - test + - stage + # - preprod + exclude: + - ${{ github.event_name == 'workflow_dispatch' && environment != github.event.inputs.environment }} + + environment: ${{ matrix.environment || github.event.inputs.environment }}-preapproved + + steps: + - name: Checkout code + uses: actions/checkout@v4.2.2 + - name: Configure kubectl + run: | + echo "${{ secrets.KUBE_CERT }}" > ca.crt + kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER} + kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }} + kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=${KUBE_NAMESPACE} + kubectl config use-context ${KUBE_CLUSTER} + env: + KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }} + KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }} + - name: Backup OpenSearch + working-directory: jobs/opensearch-backup + run: | + set -xeo pipefail + + helm install opensearch-backup . \ + --set opensearch.environment=${{ matrix.environment || github.event.inputs.environment }} + + kubectl wait jobs -l name-prefix=opensearch-backup --for=condition=complete --timeout=1h + - name: Cleanup + run: helm uninstall opensearch-backup --ignore-not-found diff --git a/jobs/opensearch-backup/templates/configmap.yaml b/jobs/opensearch-backup/templates/configmap.yaml index 5cf804b..0cb8273 100644 --- a/jobs/opensearch-backup/templates/configmap.yaml +++ b/jobs/opensearch-backup/templates/configmap.yaml @@ -2,66 +2,99 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ include "opensearch-backup.fullname" . }}-script' + name: opensearch-backup-script data: backup.sh: | #!/bin/sh set -e + echo "Starting OpenSearch backup process..." - + + # Validate required environment variables + if [ -z "$OPENSEARCH_ENDPOINT" ] || [ -z "$S3_BUCKET_NAME" ] || [ -z "$SNAPSHOT_REPOSITORY" ] || [ -z "$INDICES" ] || [ -z "$REGION" ]; then + echo "Error: Required environment variables are not set" + echo "Required variables: OPENSEARCH_ENDPOINT, S3_BUCKET_NAME, SNAPSHOT_REPOSITORY, INDICES, REGION" + exit 1 + fi + # Set timestamp for snapshot name TIMESTAMP=$(date +%Y%m%d-%H%M%S) - SNAPSHOT_NAME="${ENVIRONMENT}-${SNAPSHOT_PREFIX}-${TIMESTAMP}" - - echo "Creating snapshot: ${SNAPSHOT_NAME}" - echo "Repository: ${SNAPSHOT_REPOSITORY}" - echo "Indices to backup: ${INDICES}" - + SNAPSHOT_NAME="${SNAPSHOT_PREFIX:-backup}-${ENVIRONMENT:-default}-${TIMESTAMP}" + + echo "Creating snapshot: $SNAPSHOT_NAME" + echo "Repository: $SNAPSHOT_REPOSITORY" + echo "Indices to backup: $INDICES" + echo "---" + # Check if repository exists REPO_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY") - + if [ "$REPO_CHECK" = "404" ]; then echo "Repository does not exist. Creating snapshot repository..." - curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{ + RESPONSE=$(curl -s -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{ \"type\": \"s3\", \"settings\": { \"bucket\": \"$S3_BUCKET_NAME\", \"region\": \"$REGION\", \"role_arn\": \"$SNAPSHOT_ROLE_ARN\" } - }" - # Wait a moment for repository creation - sleep 5 + }") + if echo "$RESPONSE" | grep -q '"acknowledged":true'; then + echo "Repository created successfully" + else + echo "Failed to create repository: $RESPONSE" + exit 1 + fi + echo "---" else - echo "Repository already exists. Proceeding with backup..." + echo "Repository already exists" + echo "---" fi - + # Create the snapshot echo "Creating snapshot..." - curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME" -H 'Content-Type: application/json' -d "{ - \"indices\": \"$INDICES\", - \"include_global_state\": false - }" - + RESPONSE=$(curl -s -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME" -H 'Content-Type: application/json' -d "{ + \"indices\": \"$INDICES\", + \"include_global_state\": false + }") + + if ! echo "$RESPONSE" | grep -q '"accepted":true'; then + echo "Failed to create snapshot: $RESPONSE" + exit 1 + fi + # Monitor snapshot progress echo "Monitoring snapshot progress..." while true; do - STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_status" | grep -o '"state":"[^"]*"' | cut -d'"' -f4) - - if [ "$STATUS" = "SUCCESS" ]; then - echo "Snapshot completed successfully" + CURRENT_TIME=$(date "+%Y-%m-%d %H:%M:%S") + SNAPSHOT_STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_status") + STATE=$(echo "$SNAPSHOT_STATUS" | grep -o '"state":"[^"]*"' | cut -d'"' -f4) + + if [ "$STATE" = "SUCCESS" ]; then + echo "[$CURRENT_TIME] Snapshot completed successfully" break - elif [ "$STATUS" = "FAILED" ]; then - echo "Snapshot failed" + elif [ "$STATE" = "FAILED" ]; then + echo "[$CURRENT_TIME] Snapshot failed" exit 1 - else - echo "Snapshot in progress... (Status: $STATUS)" + else + echo "[$CURRENT_TIME] Snapshot in progress... (Status: $STATE)" sleep 10 - fi + fi done - - # List snapshots to verify - echo "Listing snapshots in repository..." - curl -XGET "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/_all" - + echo "---" + + # Get final snapshot details + echo "Snapshot Details:" + SNAPSHOT_INFO=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME") + + # Parse and display relevant information + INDICES_COUNT=$(echo "$SNAPSHOT_INFO" | grep -o '"indices":\[[^]]*\]' | grep -o ',' | wc -l) + INDICES_COUNT=$((INDICES_COUNT + 1)) + START_TIME=$(echo "$SNAPSHOT_INFO" | grep -o '"start_time":"[^"]*"' | cut -d'"' -f4) + + echo "- Name: $SNAPSHOT_NAME" + echo "- Start Time: $START_TIME" + echo "- Indices Backed Up: $INDICES_COUNT" + echo "---" + echo "Backup process completed successfully" diff --git a/jobs/opensearch-backup/templates/job.yaml b/jobs/opensearch-backup/templates/job.yaml index 4162d87..64013bd 100644 --- a/jobs/opensearch-backup/templates/job.yaml +++ b/jobs/opensearch-backup/templates/job.yaml @@ -2,48 +2,55 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ include "opensearch-backup.fullname" . }} + name: opensearch-backup spec: template: spec: serviceAccountName: "{{ $.Values.opensearch.serviceAccountPrefix }}-{{ $.Values.opensearch.environment }}" - containers: opensearch-backup - image: alpine/curl:latest - command: - - /bin/sh - - /scripts/backup.sh - env: - - name: OPENSEARCH_ENDPOINT - valueFrom: - secretKeyRef: - name: {{ $.Values.opensearch.endpointSecretName }} - key: {{ $.Values.opensearch.endpointSecretKey }} - - name: SNAPSHOT_ROLE_ARN - valueFrom: - secretKeyRef: - name: {{ $.Values.opensearch.endpointSecretName }} - key: {{ $.Values.opensearch.snapshotRoleArnKey }} - - name: S3_BUCKET_NAME - valueFrom: - secretKeyRef: - name: {{ $.Values.opensearch.s3BucketSecretName }} - key: {{ $.Values.opensearch.s3BucketNameKey }} - - name: SNAPSHOT_REPOSITORY - value: {{ $.Values.opensearch.repository }} - - name: INDICES - value: {{ $.Values.opensearch.indices }} - - name: SNAPSHOT_PREFIX - value: {{ $.Values.opensearch.snapshotPrefix }} - - name: ENVIRONMENT - value: {{ $.Values.opensearch.environment }} - - name: REGION - value: {{ $.Values.opensearch.region }} - volumeMounts: - - name: script - mountPath: /scripts + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + containers: + - name: opensearch-backup + image: ghcr.io/ministryofjustice/hmpps-delius-alfresco-utils:latest + command: + - /bin/sh + - /scripts/backup.sh + env: + - name: OPENSEARCH_ENDPOINT + valueFrom: + secretKeyRef: + name: {{ $.Values.opensearch.endpointSecretName }} + key: {{ $.Values.opensearch.endpointSecretKey }} + - name: S3_BUCKET_NAME + valueFrom: + secretKeyRef: + name: {{ $.Values.opensearch.s3BucketSecretName }} + key: {{ $.Values.opensearch.s3BucketNameKey }} + - name: SNAPSHOT_ROLE_ARN + valueFrom: + secretKeyRef: + name: {{ $.Values.opensearch.endpointSecretName }} + key: {{ $.Values.opensearch.snapshotRoleArnKey }} + - name: SNAPSHOT_REPOSITORY + value: "{{ $.Values.opensearch.repository }}" + - name: INDICES + value: "{{ $.Values.opensearch.indices }}" + - name: SNAPSHOT_PREFIX + value: "{{ $.Values.opensearch.snapshotPrefix }}" + - name: ENVIRONMENT + value: "{{ $.Values.opensearch.environment }}" + - name: REGION + value: "{{ $.Values.opensearch.region }}" + volumeMounts: + - name: script + mountPath: /scripts volumes: - name: script configMap: - name: '{{ include "opensearch-backup.fullname" . }}-script' + name: opensearch-backup-script defaultMode: 0755 - restartPolicy: Never + restartPolicy: Never \ No newline at end of file diff --git a/jobs/opensearch-backup/values.yaml b/jobs/opensearch-backup/values.yaml index 2616451..18ad57a 100644 --- a/jobs/opensearch-backup/values.yaml +++ b/jobs/opensearch-backup/values.yaml @@ -1,7 +1,7 @@ --- opensearch: repository: "daily-backups" - indices: "*" # Backup all indices by default + indices: "alfresco" snapshotPrefix: "backup" serviceAccountPrefix: "hmpps-migration" endpointSecretName: "opensearch-output" @@ -10,4 +10,4 @@ opensearch: s3BucketSecretName: "s3-opensearch-snapshots-bucket-output" s3BucketNameKey: "BUCKET_NAME" region: "eu-west-2" - envrionment: "" # Default empty, will be set by GitHub Action + environment: "" # Default empty, will be set by GitHub Action