NIT-1169 temporary data migration helpers

jobs/migrate-db/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+appVersion: 0.1
+version: 0.0.1
+description: A quickly thrown together Helm chart for deploying a job to migrate S3 data
+name: migrate-s3

jobs/migrate-db/templates/job.yaml

@@ -0,0 +1,102 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: migrate-db-script
+data:
+  entrypoint.sh: |-
+    #!/bin/bash
+    set -e
+    echo "${SRC_DB_HOST}:5432:${SRC_DB_NAME}:${SRC_DB_USER}:${SRC_DB_PASS}" > ~/.pgpass
+    echo "${DST_DB_HOST}:5432:${DST_DB_NAME}:${DST_DB_USER}:${DST_DB_PASS}" >> ~/.pgpass
+    chmod 0600 ~/.pgpass
+    set -x
+
+    pg_dump --jobs=4 --host="$SRC_DB_HOST" --username="$SRC_DB_USER" --dbname="$SRC_DB_NAME" --no-owner --no-privileges --verbose --format=directory --file=/tmp/db-dump
+    pg_restore --jobs=4 --host="$DST_DB_HOST" --username="$DST_DB_USER" --dbname="$DST_DB_NAME" --clean --if-exists --no-owner --no-privileges --verbose /tmp/db-dump
+    rm -rv /tmp/db-dump ~/.pgpass
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: migrate-db
+spec:
+  template:
+    spec:
+      containers:
+      - name: migrate-db
+        image: postgres:14
+        imagePullPolicy: IfNotPresent
+        resources:
+          limits:
+            cpu: 4
+            memory: 2Gi
+        command:
+        - /bin/entrypoint.sh
+        env:
+        - name: SRC_DB_NAME
+          valueFrom:
+            secretKeyRef:
+              name: legacy-rds-instance
+              key: DATABASE_NAME
+        - name: SRC_DB_USER
+          valueFrom:
+            secretKeyRef:
+              name: legacy-rds-instance
+              key: DATABASE_USERNAME
+        - name: SRC_DB_PASS
+          valueFrom:
+            secretKeyRef:
+              name: legacy-rds-instance
+              key: DATABASE_PASSWORD
+        - name: SRC_DB_HOST
+          valueFrom:
+            secretKeyRef:
+              name: legacy-rds-instance
+              key: RDS_INSTANCE_ADDRESS
+        - name: DST_DB_NAME
+          valueFrom:
+            secretKeyRef:
+              name: rds-instance-output
+              key: DATABASE_NAME
+        - name: DST_DB_USER
+          valueFrom:
+            secretKeyRef:
+              name: rds-instance-output
+              key: DATABASE_USERNAME
+        - name: DST_DB_PASS
+          valueFrom:
+            secretKeyRef:
+              name: rds-instance-output
+              key: DATABASE_PASSWORD
+        - name: DST_DB_HOST
+          valueFrom:
+            secretKeyRef:
+              name: rds-instance-output
+              key: RDS_INSTANCE_ADDRESS
+        volumeMounts:
+        - name: migrate-db-script
+          mountPath: /bin/entrypoint.sh
+          readOnly: true
+          subPath: entrypoint.sh
+        securityContext:
+          allowPrivilegeEscalation: false
+          privileged: false
+          readOnlyRootFilesystem: false
+          runAsNonRoot: true
+          runAsUser: 999
+          capabilities:
+            drop:
+              - ALL
+          seccompProfile:
+            type: RuntimeDefault
+      serviceAccount: hmpps-migration-{{ .Values.environment }}
+      serviceAccountName: hmpps-migration-{{ .Values.environment }}
+      restartPolicy: Never
+      volumes:
+      - name: migrate-db-script
+        configMap:
+          name: migrate-db-script
+          defaultMode: 0755
+  backoffLimit: 0
+...

jobs/migrate-db/values_stage.yaml

@@ -0,0 +1 @@
+environment: stage

jobs/migrate-s3/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+appVersion: 0.1
+version: 0.0.1
+description: A quickly thrown together Helm chart for deploying a job to migrate S3 data
+name: migrate-s3

jobs/migrate-s3/templates/job.yaml

@@ -0,0 +1,72 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: migrate-s3-script
+data:
+  entrypoint.sh: |-
+    #!/bin/sh
+    set -xe
+
+    aws configure set default.s3.max_concurrent_requests 2000
+    aws configure set default.s3.use_accelerate_endpoint true
+
+    aws s3 sync s3://$SRC_BUCKET/$DIR s3://$DST_BUCKET/$DIR --delete --only-show-errors
+
+    echo sync of $DIR directory completed
+{{- range .Values.dirs }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: migrate-s3-{{ . | toString | replace "/" "-" }}
+spec:
+  template:
+    spec:
+      containers:
+      - name: migrate-s3
+        image: 754256621582.dkr.ecr.eu-west-2.amazonaws.com/webops/cloud-platform-service-pod:c5f69b4624b956248001fa7c173c89a0556a457e
+        imagePullPolicy: IfNotPresent
+        resources:
+          limits:
+            cpu: 4
+            memory: 8Gi
+        command:
+        - /bin/entrypoint.sh
+        env:
+        - name: SRC_BUCKET
+          value: {{ $.Values.srcBucket }}
+        - name: DST_BUCKET
+          valueFrom:
+            secretKeyRef:
+              name: s3-bucket-output
+              key: BUCKET_NAME
+        - name: DIR
+          value: {{ . | quote }}
+        volumeMounts:
+        - name: migrate-s3-script
+          mountPath: /bin/entrypoint.sh
+          readOnly: true
+          subPath: entrypoint.sh
+        securityContext:
+          allowPrivilegeEscalation: false
+          privileged: false
+          readOnlyRootFilesystem: false
+          runAsNonRoot: true
+          runAsUser: 1001
+          capabilities:
+            drop:
+              - ALL
+          seccompProfile:
+            type: RuntimeDefault
+      serviceAccount: hmpps-migration-{{ $.Values.environment }}
+      serviceAccountName: hmpps-migration-{{ $.Values.environment }}
+      restartPolicy: OnFailure
+      volumes:
+      - name: migrate-s3-script
+        configMap:
+          name: migrate-s3-script
+          defaultMode: 0755
+  backoffLimit: 10
+{{- end }}
+...

jobs/migrate-s3/values_stage.yaml

@@ -0,0 +1,13 @@
+environment: stage
+srcBucket: tf-eu-west-2-hmpps-delius-stage-alfresco-storage-s3bucket
+dirs:
+  - 2019
+  - 2020
+  - 2021
+  - 2022
+  - 2023
+  - 2024
+  - contentstore/2016
+  - contentstore/2017
+  - contentstore/2018
+  - contentstore/2019