diff --git a/jobs/migrate-db/Chart.yaml b/jobs/migrate-db/Chart.yaml new file mode 100644 index 0000000..9a4bbde --- /dev/null +++ b/jobs/migrate-db/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: 0.1 +version: 0.0.1 +description: A quickly thrown together Helm chart for deploying a job to migrate S3 data +name: migrate-s3 diff --git a/jobs/migrate-db/templates/job.yaml b/jobs/migrate-db/templates/job.yaml new file mode 100644 index 0000000..6baddb9 --- /dev/null +++ b/jobs/migrate-db/templates/job.yaml @@ -0,0 +1,102 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: migrate-db-script +data: + entrypoint.sh: |- + #!/bin/bash + set -e + echo "${SRC_DB_HOST}:5432:${SRC_DB_NAME}:${SRC_DB_USER}:${SRC_DB_PASS}" > ~/.pgpass + echo "${DST_DB_HOST}:5432:${DST_DB_NAME}:${DST_DB_USER}:${DST_DB_PASS}" >> ~/.pgpass + chmod 0600 ~/.pgpass + set -x + + pg_dump --jobs=4 --host="$SRC_DB_HOST" --username="$SRC_DB_USER" --dbname="$SRC_DB_NAME" --no-owner --no-privileges --verbose --format=directory --file=/tmp/db-dump + pg_restore --jobs=4 --host="$DST_DB_HOST" --username="$DST_DB_USER" --dbname="$DST_DB_NAME" --clean --if-exists --no-owner --no-privileges --verbose /tmp/db-dump + rm -rv /tmp/db-dump ~/.pgpass +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: migrate-db +spec: + template: + spec: + containers: + - name: migrate-db + image: postgres:14 + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 4 + memory: 2Gi + command: + - /bin/entrypoint.sh + env: + - name: SRC_DB_NAME + valueFrom: + secretKeyRef: + name: legacy-rds-instance + key: DATABASE_NAME + - name: SRC_DB_USER + valueFrom: + secretKeyRef: + name: legacy-rds-instance + key: DATABASE_USERNAME + - name: SRC_DB_PASS + valueFrom: + secretKeyRef: + name: legacy-rds-instance + key: DATABASE_PASSWORD + - name: SRC_DB_HOST + valueFrom: + secretKeyRef: + name: legacy-rds-instance + key: RDS_INSTANCE_ADDRESS + - name: DST_DB_NAME + valueFrom: + secretKeyRef: + name: rds-instance-output + key: DATABASE_NAME + - name: DST_DB_USER + valueFrom: + secretKeyRef: + name: rds-instance-output + key: DATABASE_USERNAME + - name: DST_DB_PASS + valueFrom: + secretKeyRef: + name: rds-instance-output + key: DATABASE_PASSWORD + - name: DST_DB_HOST + valueFrom: + secretKeyRef: + name: rds-instance-output + key: RDS_INSTANCE_ADDRESS + volumeMounts: + - name: migrate-db-script + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + serviceAccount: hmpps-migration-{{ .Values.environment }} + serviceAccountName: hmpps-migration-{{ .Values.environment }} + restartPolicy: Never + volumes: + - name: migrate-db-script + configMap: + name: migrate-db-script + defaultMode: 0755 + backoffLimit: 0 +... diff --git a/jobs/migrate-db/values_stage.yaml b/jobs/migrate-db/values_stage.yaml new file mode 100644 index 0000000..01dd174 --- /dev/null +++ b/jobs/migrate-db/values_stage.yaml @@ -0,0 +1 @@ +environment: stage diff --git a/jobs/migrate-s3/Chart.yaml b/jobs/migrate-s3/Chart.yaml new file mode 100644 index 0000000..9a4bbde --- /dev/null +++ b/jobs/migrate-s3/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: 0.1 +version: 0.0.1 +description: A quickly thrown together Helm chart for deploying a job to migrate S3 data +name: migrate-s3 diff --git a/jobs/migrate-s3/templates/job.yaml b/jobs/migrate-s3/templates/job.yaml new file mode 100644 index 0000000..669e5b8 --- /dev/null +++ b/jobs/migrate-s3/templates/job.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: migrate-s3-script +data: + entrypoint.sh: |- + #!/bin/sh + set -xe + + aws configure set default.s3.max_concurrent_requests 2000 + aws configure set default.s3.use_accelerate_endpoint true + + aws s3 sync s3://$SRC_BUCKET/$DIR s3://$DST_BUCKET/$DIR --delete --only-show-errors + + echo sync of $DIR directory completed +{{- range .Values.dirs }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: migrate-s3-{{ . | toString | replace "/" "-" }} +spec: + template: + spec: + containers: + - name: migrate-s3 + image: 754256621582.dkr.ecr.eu-west-2.amazonaws.com/webops/cloud-platform-service-pod:c5f69b4624b956248001fa7c173c89a0556a457e + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 4 + memory: 8Gi + command: + - /bin/entrypoint.sh + env: + - name: SRC_BUCKET + value: {{ $.Values.srcBucket }} + - name: DST_BUCKET + valueFrom: + secretKeyRef: + name: s3-bucket-output + key: BUCKET_NAME + - name: DIR + value: {{ . | quote }} + volumeMounts: + - name: migrate-s3-script + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 1001 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + serviceAccount: hmpps-migration-{{ $.Values.environment }} + serviceAccountName: hmpps-migration-{{ $.Values.environment }} + restartPolicy: OnFailure + volumes: + - name: migrate-s3-script + configMap: + name: migrate-s3-script + defaultMode: 0755 + backoffLimit: 10 +{{- end }} +... diff --git a/jobs/migrate-s3/values_stage.yaml b/jobs/migrate-s3/values_stage.yaml new file mode 100644 index 0000000..d2b0a3a --- /dev/null +++ b/jobs/migrate-s3/values_stage.yaml @@ -0,0 +1,13 @@ +environment: stage +srcBucket: tf-eu-west-2-hmpps-delius-stage-alfresco-storage-s3bucket +dirs: + - 2019 + - 2020 + - 2021 + - 2022 + - 2023 + - 2024 + - contentstore/2016 + - contentstore/2017 + - contentstore/2018 + - contentstore/2019