From 0ae2bd0df669350c27cfea1127cbb5e37260018c Mon Sep 17 00:00:00 2001
From: George Taylor <george.taylor@digital.justice.gov.uk>
Date: Wed, 9 Oct 2024 12:23:08 +0100
Subject: [PATCH] new migration job

---
 .github/workflows/data-refresh.yaml |  6 +--
 .github/workflows/migrate-s3.yaml   | 84 +++++++++++++++++++++++++++++
 jobs/migrate-s3/values_preprod.yaml | 13 -----
 jobs/migrate-s3/values_stage.yaml   | 13 -----
 4 files changed, 87 insertions(+), 29 deletions(-)
 create mode 100644 .github/workflows/migrate-s3.yaml
 delete mode 100644 jobs/migrate-s3/values_preprod.yaml
 delete mode 100644 jobs/migrate-s3/values_stage.yaml

diff --git a/.github/workflows/data-refresh.yaml b/.github/workflows/data-refresh.yaml
index a26d8e3..6f37b64 100644
--- a/.github/workflows/data-refresh.yaml
+++ b/.github/workflows/data-refresh.yaml
@@ -57,7 +57,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4.1.1
 
-      - name: Install Kubernetes
+      - name: Install kubectl
         uses: azure/setup-kubectl@v3.2
         with:
           version: 'v1.26.0' # default is latest stable
@@ -99,7 +99,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4.1.1
 
-      - name: Install Kubernetes
+      - name: Install kubectl
         uses: azure/setup-kubectl@v3.2
         with:
           version: 'v1.26.0' # default is latest stable
@@ -133,7 +133,7 @@ jobs:
           SERVICE_POD_NAME=$(kubectl get pod -l app=$SERVICE_POD_DEPLOYMENT -o jsonpath="{.items[0].metadata.name}")
 
           SRC_BUCKET=$(kubectl get secrets s3-bucket-output -o jsonpath='{.data.BUCKET_NAME}' | base64 -d)
-          
+
           DIRS=""
           for prefix in $(aws s3api list-objects-v2 --bucket $SRC_BUCKET --delimiter '/' --query 'CommonPrefixes[*].Prefix' --output text); do
               DIR=$(aws s3api list-objects-v2 --bucket $SRC_BUCKET --prefix "$prefix" --delimiter '/' --query 'CommonPrefixes[*].Prefix' --output text)
diff --git a/.github/workflows/migrate-s3.yaml b/.github/workflows/migrate-s3.yaml
new file mode 100644
index 0000000..efe89ce
--- /dev/null
+++ b/.github/workflows/migrate-s3.yaml
@@ -0,0 +1,84 @@
+---
+name: Alfresco s3 Migration
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to copy data to
+        required: true
+        type: choice
+        options:
+          - poc
+          - dev
+          - test
+          - stage
+          - preprod
+          - prod
+
+permissions:
+  contents: read
+
+jobs:
+  refresh-s3:
+    name: Refresh S3
+    runs-on: ubuntu-22.04
+    environment:
+      name: ${{ github.event.inputs.environment }}
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - name: Install kubectl
+        uses: azure/setup-kubectl@v3.2
+        with:
+          version: 'v1.26.0' # default is latest stable
+        id: kubectl_install
+
+      - uses: azure/setup-helm@v4.2.0
+        with:
+           version: 'v3.15.3' # default is latest (stable)
+        id: install
+
+      - name: Configure kubectl
+        run: |
+          echo "${{ secrets.KUBE_CERT }}" > ca.crt
+          kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER}
+          kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }}
+          kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=${KUBE_NAMESPACE}
+          kubectl config use-context ${KUBE_CLUSTER}
+        env:
+          KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }}
+          KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }}
+      
+      - name: Uninstall S3 Refresh chart
+        run: helm uninstall refresh-s3 --ignore-not-found
+
+      - name: S3 migration
+        working-directory: jobs/migrate-s3
+        run: |
+          set -xeo pipefail
+
+          SERVICE_POD_DEPLOYMENT=$(kubectl get deployment -l app=service-pod -o jsonpath="{.items[0].metadata.name}")
+          SERVICE_POD_NAME=$(kubectl get pod -l app=$SERVICE_POD_DEPLOYMENT -o jsonpath="{.items[0].metadata.name}")
+
+          SRC_BUCKET=${{ vars.MIGRATION_SRC_BUCKET }}
+          
+          DIRS=""
+          for prefix in $(aws s3api list-objects-v2 --bucket $SRC_BUCKET --delimiter '/' --query 'CommonPrefixes[*].Prefix' --output text); do
+              DIR=$(aws s3api list-objects-v2 --bucket $SRC_BUCKET --prefix "$prefix" --delimiter '/' --query 'CommonPrefixes[*].Prefix' --output text)
+              #append to DIRS comma separated
+              DIRS+="${DIR},"
+          done
+          DIRS=${DIRS%,}
+
+
+          echo "DIRS: $DIRS"
+
+          # helm install migrate-s3 . \
+          #   --set environment=${{ github.event.inputs.environment }} \
+          #   --set directories="{${DIRS}}"
+
+          # kubectl wait jobs -l name-prefix=migrate-s3 --for=condition=complete --timeout 10h
+
+      - name: Uninstall S3 Refresh chart
+        run: helm uninstall refresh-s3 --ignore-not-found
diff --git a/jobs/migrate-s3/values_preprod.yaml b/jobs/migrate-s3/values_preprod.yaml
deleted file mode 100644
index 53c72e2..0000000
--- a/jobs/migrate-s3/values_preprod.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-environment: preprod
-srcBucket: tf-eu-west-2-hmpps-delius-pre-prod-alfresco-storage-s3bucket
-dirs:
-  - 2019
-  - 2020
-  - 2021
-  - 2022
-  - 2023
-  - 2024
-  - contentstore/2016
-  - contentstore/2017
-  - contentstore/2018
-  - contentstore/2019
diff --git a/jobs/migrate-s3/values_stage.yaml b/jobs/migrate-s3/values_stage.yaml
deleted file mode 100644
index d2b0a3a..0000000
--- a/jobs/migrate-s3/values_stage.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-environment: stage
-srcBucket: tf-eu-west-2-hmpps-delius-stage-alfresco-storage-s3bucket
-dirs:
-  - 2019
-  - 2020
-  - 2021
-  - 2022
-  - 2023
-  - 2024
-  - contentstore/2016
-  - contentstore/2017
-  - contentstore/2018
-  - contentstore/2019