diff --git a/.github/workflows/data-refresh.yaml b/.github/workflows/data-refresh.yaml index 2bdf0be..fac366c 100644 --- a/.github/workflows/data-refresh.yaml +++ b/.github/workflows/data-refresh.yaml @@ -64,13 +64,16 @@ jobs: KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }} KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }} - - name: Start DB Refresh Job + - name: DB Refresh + working-directory: jobs/refresh-db run: | - kubectl apply -f jobs/refresh-db.yaml - kubectl wait --timeout 10m --for=condition=complete job/refresh-db + helm install refresh-db . \ + --set sourceEnvironment=${{ github.event.inputs.source_env }} \ + --set destinationEnvironment=${{ github.event.inputs.destination_env }} + kubectl wait job refresh-db --for=condition=complete --timeout 10h - - name: Delete DB Refresh Job - run: kubectl delete job refresh-db + - name: Uninstall DB Refresh chart + run: helm uninstall refresh-db refresh-s3: name: Refresh S3 @@ -92,13 +95,27 @@ jobs: KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }} KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }} - - name: Start S3 Refresh Job + - name: S3 Refresh + working-directory: jobs/refresh-s3 run: | - kubectl apply -f jobs/refresh-s3.yaml - kubectl wait --timeout 10m --for=condition=complete job/refresh-s3 + set -xeo pipefail - - name: Delete S3 Refresh Job - run: kubectl delete job refresh-s3 + SERVICE_POD_DEPLOYMENT=$(kubectl get deployment -l app=service-pod -o jsonpath="{.items[0].metadata.name}") + SERVICE_POD_NAME=$(kubectl get pod -l app=$SERVICE_POD_DEPLOYMENT -o jsonpath="{.items[0].metadata.name}") + + SRC_BUCKET=$(kubectl get secrets s3-bucket-output -o jsonpath='{.data.BUCKET_NAME}' | base64 -d) + + DIRS=$(kubectl exec $SERVICE_POD_NAME -- aws s3 ls $SRC_BUCKET | grep -v contentstore | awk -F ' ' '{print $2}' | tr -d '/' | tr '\n' ',') + + helm install refresh-s3 . \ + --set sourceEnvironment=${{ github.event.inputs.source_env }} \ + --set destinationEnvironment=${{ github.event.inputs.destination_env }} \ + --set directories="{${DIRS:0:-1}}" + + kubectl wait jobs -l name-prefix=refresh-s3 --for=condition=complete --timeout 10h + + - name: Uninstall S3 Refresh chart + run: helm uninstall refresh-s3 wipe-solr-data: name: Wipe Solr Data @@ -151,6 +168,10 @@ jobs: - name: Stop ${{ github.event.inputs.destination_env }} Environment run: | - kubectl scale deployment alfresco-content-services-alfresco-cs-repository --replicas=1 - kubectl scale deployment alfresco-content-services-alfresco-cs-share --replicas=1 + apt update && apt install -y jq + + HELM_VALUES=$(helm get values alfresco-content-services -o json) + + kubectl scale deployment alfresco-content-services-alfresco-cs-repository --replicas=$(echo $HELM_VALUES | jq '.repository.replicaCount') + kubectl scale deployment alfresco-content-services-alfresco-cs-share --replicas=$(echo $HELM_VALUES | jq '.share.replicaCount') kubectl scale deployment alfresco-content-services-alfresco-search-solr --replicas=1 diff --git a/jobs/migrate-db/Chart.yaml b/jobs/migrate-db/Chart.yaml new file mode 100644 index 0000000..2d1c7da --- /dev/null +++ b/jobs/migrate-db/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: 0.1 +version: 0.0.1 +description: A quickly thrown together Helm chart for deploying a job to migrate DB data +name: migrate-db diff --git a/jobs/migrate-db/templates/job.yaml b/jobs/migrate-db/templates/job.yaml new file mode 100644 index 0000000..6baddb9 --- /dev/null +++ b/jobs/migrate-db/templates/job.yaml @@ -0,0 +1,102 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: migrate-db-script +data: + entrypoint.sh: |- + #!/bin/bash + set -e + echo "${SRC_DB_HOST}:5432:${SRC_DB_NAME}:${SRC_DB_USER}:${SRC_DB_PASS}" > ~/.pgpass + echo "${DST_DB_HOST}:5432:${DST_DB_NAME}:${DST_DB_USER}:${DST_DB_PASS}" >> ~/.pgpass + chmod 0600 ~/.pgpass + set -x + + pg_dump --jobs=4 --host="$SRC_DB_HOST" --username="$SRC_DB_USER" --dbname="$SRC_DB_NAME" --no-owner --no-privileges --verbose --format=directory --file=/tmp/db-dump + pg_restore --jobs=4 --host="$DST_DB_HOST" --username="$DST_DB_USER" --dbname="$DST_DB_NAME" --clean --if-exists --no-owner --no-privileges --verbose /tmp/db-dump + rm -rv /tmp/db-dump ~/.pgpass +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: migrate-db +spec: + template: + spec: + containers: + - name: migrate-db + image: postgres:14 + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 4 + memory: 2Gi + command: + - /bin/entrypoint.sh + env: + - name: SRC_DB_NAME + valueFrom: + secretKeyRef: + name: legacy-rds-instance + key: DATABASE_NAME + - name: SRC_DB_USER + valueFrom: + secretKeyRef: + name: legacy-rds-instance + key: DATABASE_USERNAME + - name: SRC_DB_PASS + valueFrom: + secretKeyRef: + name: legacy-rds-instance + key: DATABASE_PASSWORD + - name: SRC_DB_HOST + valueFrom: + secretKeyRef: + name: legacy-rds-instance + key: RDS_INSTANCE_ADDRESS + - name: DST_DB_NAME + valueFrom: + secretKeyRef: + name: rds-instance-output + key: DATABASE_NAME + - name: DST_DB_USER + valueFrom: + secretKeyRef: + name: rds-instance-output + key: DATABASE_USERNAME + - name: DST_DB_PASS + valueFrom: + secretKeyRef: + name: rds-instance-output + key: DATABASE_PASSWORD + - name: DST_DB_HOST + valueFrom: + secretKeyRef: + name: rds-instance-output + key: RDS_INSTANCE_ADDRESS + volumeMounts: + - name: migrate-db-script + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + serviceAccount: hmpps-migration-{{ .Values.environment }} + serviceAccountName: hmpps-migration-{{ .Values.environment }} + restartPolicy: Never + volumes: + - name: migrate-db-script + configMap: + name: migrate-db-script + defaultMode: 0755 + backoffLimit: 0 +... diff --git a/jobs/migrate-db/values_stage.yaml b/jobs/migrate-db/values_stage.yaml new file mode 100644 index 0000000..01dd174 --- /dev/null +++ b/jobs/migrate-db/values_stage.yaml @@ -0,0 +1 @@ +environment: stage diff --git a/jobs/migrate-s3/Chart.yaml b/jobs/migrate-s3/Chart.yaml new file mode 100644 index 0000000..9a4bbde --- /dev/null +++ b/jobs/migrate-s3/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: 0.1 +version: 0.0.1 +description: A quickly thrown together Helm chart for deploying a job to migrate S3 data +name: migrate-s3 diff --git a/jobs/migrate-s3/templates/job.yaml b/jobs/migrate-s3/templates/job.yaml new file mode 100644 index 0000000..669e5b8 --- /dev/null +++ b/jobs/migrate-s3/templates/job.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: migrate-s3-script +data: + entrypoint.sh: |- + #!/bin/sh + set -xe + + aws configure set default.s3.max_concurrent_requests 2000 + aws configure set default.s3.use_accelerate_endpoint true + + aws s3 sync s3://$SRC_BUCKET/$DIR s3://$DST_BUCKET/$DIR --delete --only-show-errors + + echo sync of $DIR directory completed +{{- range .Values.dirs }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: migrate-s3-{{ . | toString | replace "/" "-" }} +spec: + template: + spec: + containers: + - name: migrate-s3 + image: 754256621582.dkr.ecr.eu-west-2.amazonaws.com/webops/cloud-platform-service-pod:c5f69b4624b956248001fa7c173c89a0556a457e + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 4 + memory: 8Gi + command: + - /bin/entrypoint.sh + env: + - name: SRC_BUCKET + value: {{ $.Values.srcBucket }} + - name: DST_BUCKET + valueFrom: + secretKeyRef: + name: s3-bucket-output + key: BUCKET_NAME + - name: DIR + value: {{ . | quote }} + volumeMounts: + - name: migrate-s3-script + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 1001 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + serviceAccount: hmpps-migration-{{ $.Values.environment }} + serviceAccountName: hmpps-migration-{{ $.Values.environment }} + restartPolicy: OnFailure + volumes: + - name: migrate-s3-script + configMap: + name: migrate-s3-script + defaultMode: 0755 + backoffLimit: 10 +{{- end }} +... diff --git a/jobs/migrate-s3/values_stage.yaml b/jobs/migrate-s3/values_stage.yaml new file mode 100644 index 0000000..d2b0a3a --- /dev/null +++ b/jobs/migrate-s3/values_stage.yaml @@ -0,0 +1,13 @@ +environment: stage +srcBucket: tf-eu-west-2-hmpps-delius-stage-alfresco-storage-s3bucket +dirs: + - 2019 + - 2020 + - 2021 + - 2022 + - 2023 + - 2024 + - contentstore/2016 + - contentstore/2017 + - contentstore/2018 + - contentstore/2019 diff --git a/jobs/refresh-db/Chart.yaml b/jobs/refresh-db/Chart.yaml new file mode 100644 index 0000000..c09d657 --- /dev/null +++ b/jobs/refresh-db/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: 0.1 +version: 0.0.1 +description: Job to refresh DB data +name: refresh-db diff --git a/jobs/refresh-db.yaml b/jobs/refresh-db/templates/job.yaml similarity index 82% rename from jobs/refresh-db.yaml rename to jobs/refresh-db/templates/job.yaml index 07f95f4..617b077 100644 --- a/jobs/refresh-db.yaml +++ b/jobs/refresh-db/templates/job.yaml @@ -27,6 +27,10 @@ spec: - name: refresh-db image: postgres:14 imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 4 + memory: 2Gi command: - /bin/entrypoint.sh env: @@ -53,22 +57,22 @@ spec: - name: DST_DB_NAME valueFrom: secretKeyRef: - name: rds-instance-output-poc + name: rds-instance-output-{{ .Values.destinationEnvironment }} key: DATABASE_NAME - name: DST_DB_USER valueFrom: secretKeyRef: - name: rds-instance-output-poc + name: rds-instance-output-{{ .Values.destinationEnvironment }} key: DATABASE_USERNAME - name: DST_DB_PASS valueFrom: secretKeyRef: - name: rds-instance-output-poc + name: rds-instance-output-{{ .Values.destinationEnvironment }} key: DATABASE_PASSWORD - name: DST_DB_HOST valueFrom: secretKeyRef: - name: rds-instance-output-poc + name: rds-instance-output-{{ .Values.destinationEnvironment }} key: RDS_INSTANCE_ADDRESS volumeMounts: - name: refresh-db-script @@ -79,7 +83,13 @@ spec: allowPrivilegeEscalation: false privileged: false readOnlyRootFilesystem: false + runAsNonRoot: true runAsUser: 999 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault serviceAccount: hmpps-migration-development serviceAccountName: hmpps-migration-development restartPolicy: Never diff --git a/jobs/refresh-s3/Chart.yaml b/jobs/refresh-s3/Chart.yaml new file mode 100644 index 0000000..4300a06 --- /dev/null +++ b/jobs/refresh-s3/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: 0.1 +version: 0.0.1 +description: Jobs to refresh S3 data +name: refresh-s3 diff --git a/jobs/refresh-s3.yaml b/jobs/refresh-s3/templates/job.yaml similarity index 65% rename from jobs/refresh-s3.yaml rename to jobs/refresh-s3/templates/job.yaml index 3b01b35..29552e9 100644 --- a/jobs/refresh-s3.yaml +++ b/jobs/refresh-s3/templates/job.yaml @@ -9,13 +9,21 @@ data: set -xe aws configure set default.s3.max_concurrent_requests 2000 + # aws configure set default.s3.use_accelerate_endpoint true - aws s3 sync s3://$SRC_BUCKET/ s3://$DST_BUCKET/ --delete + aws s3 sync s3://$SRC_BUCKET/$DIR s3://$DST_BUCKET/$DIR --delete --only-show-errors + + set +x + echo sync of $DIR directory completed + +{{- range .Values.directories }} --- apiVersion: batch/v1 kind: Job metadata: - name: refresh-s3 + name: refresh-s3-{{ . | toString | replace "/" "-" }} + labels: + name-prefix: refresh-s3 spec: template: spec: @@ -23,6 +31,10 @@ spec: - name: refresh-s3 image: 754256621582.dkr.ecr.eu-west-2.amazonaws.com/webops/cloud-platform-service-pod:c5f69b4624b956248001fa7c173c89a0556a457e imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 4 + memory: 8Gi command: - /bin/entrypoint.sh env: @@ -34,8 +46,10 @@ spec: - name: DST_BUCKET valueFrom: secretKeyRef: - name: s3-bucket-output-poc + name: s3-bucket-output-{{ $.Values.destinationEnvironment }} key: BUCKET_NAME + - name: DIR + value: {{ . | quote }} volumeMounts: - name: refresh-s3-script mountPath: /bin/entrypoint.sh @@ -47,13 +61,19 @@ spec: readOnlyRootFilesystem: false runAsNonRoot: true runAsUser: 1001 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault serviceAccount: hmpps-migration-development serviceAccountName: hmpps-migration-development - restartPolicy: Never + restartPolicy: OnFailure volumes: - name: refresh-s3-script configMap: name: refresh-s3-script defaultMode: 0755 - backoffLimit: 0 + backoffLimit: 10 +{{- end }} ...