PI-1759 Add cron job to redrive DLQ messages + extend DLQ retention t…

…o 1 week (#2964)

* PI-1759 Extend DLQ retention period to 1 week

* PI-1759 Add cron job to redrive DLQ messages

* Pin base image version

* Only generate docs for projects with a tech-docs directory

.github/actions/cloud-platform-deploy/action.yml

@@ -6,7 +6,7 @@ inputs:
     description: The project to deploy
     required: true
   environment:
-    description: The environment to deploy to (dev/preprod/prod)
+    description: The environment name to deploy to (test/preprod/prod)
     required: true
   version:
     description: The version of the service to deploy

.github/actions/render-project-template/action.yml

@@ -31,7 +31,6 @@ runs:
         sed -i '/add new projects here/i \          - '"'"'["${{ inputs.project_name }}"]'"'"'' .github/workflows/access.yml
         sed -i '/add new projects here/i \          - '"'"'["${{ inputs.project_name }}"]'"'"'' .github/workflows/deploy.yml
         sed -i '/add new projects here/i \          - '"'"'["${{ inputs.project_name }}"]'"'"'' .github/workflows/docs.yml
-        sed -i '/add new projects here/i \          - ${{ inputs.project_name }}' .github/workflows/suppress-trivy.yml
         sed -i '/add new projects here/i \          - ${{ inputs.project_name }}' .github/workflows/build.yml
         sed -i '/add new projects here/i \* [${{ steps.project_name.outputs.title_case }}](https://ministryofjustice.github.io/hmpps-probation-integration-services/tech-docs/projects/${{ inputs.project_name }})' doc/tech-docs/source/services.html.md.erb
         sed 's/$SERVICE_NAME/${{ inputs.project_name }}/g' templates/runConfiguration.xml > '.idea/runConfigurations/${{ steps.project_name.outputs.underscore }}.xml'

.github/dependabot.yml

@@ -25,3 +25,8 @@ updates:
     directory: "/projects/person-search-index-from-delius/container"
     schedule:
       interval: "daily"
+
+  - package-ecosystem: "docker"
+    directory: "/projects/redrive-dead-letter-queues/container"
+    schedule:
+      interval: "monthly" # to reduce frequency of AWS CLI updates

.github/workflows/access.yml

@@ -12,40 +12,40 @@ on:
           - 'All'
           - '["approved-premises-and-delius"]'
           - '["approved-premises-and-oasys"]'
+          - '["arns-and-delius"]'
+          - '["assessment-summary-and-delius"]'
+          - '["cas3-and-delius"]'
           - '["court-case-and-delius"]'
           - '["create-and-vary-a-licence-and-delius"]'
           - '["custody-key-dates-and-delius"]'
           - '["domain-events-and-delius"]'
+          - '["dps-and-delius"]'
           - '["effective-proposal-framework-and-delius"]'
           - '["external-api-and-delius"]'
+          - '["hdc-licences-and-delius"]'
+          - '["hmpps-auth-and-delius"]'
           - '["make-recall-decisions-and-delius"]'
+          - '["manage-offences-and-delius"]'
           - '["manage-pom-cases-and-delius"]'
           - '["offender-events-and-delius"]'
+          - '["opd-and-delius"]'
           - '["pathfinder-and-delius"]'
           - '["person-search-index-from-delius"]'
           - '["pre-sentence-reports-to-delius"]'
           - '["prison-case-notes-to-probation"]'
           - '["prison-custody-status-to-delius"]'
+          - '["prison-education-and-delius"]'
+          - '["prison-identifier-and-delius"]'
+          - '["prisoner-profile-and-delius"]'
           - '["refer-and-monitor-and-delius"]'
+          - '["resettlement-passport-and-delius"]'
           - '["risk-assessment-scores-to-delius"]'
           - '["sentence-plan-and-delius"]'
           - '["sentence-plan-and-oasys"]'
           - '["soc-and-delius"]'
           - '["tier-to-delius"]'
           - '["unpaid-work-and-delius"]'
           - '["workforce-allocations-to-delius"]'
-          - '["manage-offences-and-delius"]'
-          - '["resettlement-passport-and-delius"]'
-          - '["prison-education-and-delius"]'
-          - '["opd-and-delius"]'
-          - '["hmpps-auth-and-delius"]'
-          - '["dps-and-delius"]'
-          - '["arns-and-delius"]'
-          - '["cas3-and-delius"]'
-          - '["hdc-licences-and-delius"]'
-          - '["prisoner-profile-and-delius"]'
-          - '["prison-identifier-and-delius"]'
-          - '["assessment-summary-and-delius"]'
           # ^ add new projects here
           # GitHub Actions doesn't support dynamic choices, we must add each project here to enable manual deployments
           # See https://github.com/community/community/discussions/11795

.github/workflows/build.yml

@@ -36,39 +36,39 @@ jobs:
         project:
           - approved-premises-and-delius
           - approved-premises-and-oasys
+          - arns-and-delius
+          - assessment-summary-and-delius
+          - cas3-and-delius
           - court-case-and-delius
           - create-and-vary-a-licence-and-delius
           - custody-key-dates-and-delius
           - domain-events-and-delius
+          - dps-and-delius
           - effective-proposal-framework-and-delius
           - external-api-and-delius
+          - hdc-licences-and-delius
+          - hmpps-auth-and-delius
           - make-recall-decisions-and-delius
+          - manage-offences-and-delius
           - manage-pom-cases-and-delius
           - offender-events-and-delius
+          - opd-and-delius
           - pathfinder-and-delius
           - pre-sentence-reports-to-delius
           - prison-case-notes-to-probation
           - prison-custody-status-to-delius
+          - prison-education-and-delius
+          - prison-identifier-and-delius
+          - prisoner-profile-and-delius
           - refer-and-monitor-and-delius
+          - resettlement-passport-and-delius
           - risk-assessment-scores-to-delius
           - sentence-plan-and-delius
           - sentence-plan-and-oasys
           - soc-and-delius
           - tier-to-delius
           - unpaid-work-and-delius
           - workforce-allocations-to-delius
-          - manage-offences-and-delius
-          - resettlement-passport-and-delius
-          - prison-education-and-delius
-          - opd-and-delius
-          - hmpps-auth-and-delius
-          - dps-and-delius
-          - arns-and-delius
-          - cas3-and-delius
-          - hdc-licences-and-delius
-          - prisoner-profile-and-delius
-          - prison-identifier-and-delius
-          - assessment-summary-and-delius
           # ^ add new projects here
           # GitHub Actions doesn't support dynamic choices, we must add each project here to enable manual deployments
           # See https://github.com/community/community/discussions/11795
@@ -129,6 +129,7 @@ jobs:
       matrix:
         project:
           - person-search-index-from-delius
+          - redrive-dead-letter-queues
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/deploy.yml

@@ -8,7 +8,7 @@ on:
         type: string
         required: true
       environment:
-        description: The name of the environment to deploy to
+        description: The environment name to deploy to (test/preprod/prod)
         type: string
         required: true
       version:
@@ -25,40 +25,41 @@ on:
         options:
           - '["approved-premises-and-delius"]'
           - '["approved-premises-and-oasys"]'
+          - '["arns-and-delius"]'
+          - '["assessment-summary-and-delius"]'
+          - '["cas3-and-delius"]'
           - '["court-case-and-delius"]'
           - '["create-and-vary-a-licence-and-delius"]'
           - '["custody-key-dates-and-delius"]'
           - '["domain-events-and-delius"]'
+          - '["dps-and-delius"]'
           - '["effective-proposal-framework-and-delius"]'
           - '["external-api-and-delius"]'
+          - '["hdc-licences-and-delius"]'
+          - '["hmpps-auth-and-delius"]'
           - '["make-recall-decisions-and-delius"]'
+          - '["manage-offences-and-delius"]'
           - '["manage-pom-cases-and-delius"]'
           - '["offender-events-and-delius"]'
+          - '["opd-and-delius"]'
           - '["pathfinder-and-delius"]'
           - '["person-search-index-from-delius"]'
           - '["pre-sentence-reports-to-delius"]'
           - '["prison-case-notes-to-probation"]'
           - '["prison-custody-status-to-delius"]'
+          - '["prison-education-and-delius"]'
+          - '["prison-identifier-and-delius"]'
+          - '["prisoner-profile-and-delius"]'
+          - '["redrive-dead-letter-queue"]'
           - '["refer-and-monitor-and-delius"]'
+          - '["resettlement-passport-and-delius"]'
           - '["risk-assessment-scores-to-delius"]'
           - '["sentence-plan-and-delius"]'
           - '["sentence-plan-and-oasys"]'
           - '["soc-and-delius"]'
           - '["tier-to-delius"]'
           - '["unpaid-work-and-delius"]'
           - '["workforce-allocations-to-delius"]'
-          - '["manage-offences-and-delius"]'
-          - '["resettlement-passport-and-delius"]'
-          - '["prison-education-and-delius"]'
-          - '["opd-and-delius"]'
-          - '["hmpps-auth-and-delius"]'
-          - '["dps-and-delius"]'
-          - '["arns-and-delius"]'
-          - '["cas3-and-delius"]'
-          - '["hdc-licences-and-delius"]'
-          - '["prisoner-profile-and-delius"]'
-          - '["prison-identifier-and-delius"]'
-          - '["assessment-summary-and-delius"]'
           # ^ add new projects here
           # GitHub Actions doesn't support dynamic choices, we must add each project here to enable manual deployments
           # See https://github.com/community/community/discussions/11795

.github/workflows/docs.yml

@@ -18,40 +18,40 @@ on:
           - 'All'
           - '["approved-premises-and-delius"]'
           - '["approved-premises-and-oasys"]'
+          - '["arns-and-delius"]'
+          - '["assessment-summary-and-delius"]'
+          - '["cas3-and-delius"]'
           - '["court-case-and-delius"]'
           - '["create-and-vary-a-licence-and-delius"]'
           - '["custody-key-dates-and-delius"]'
           - '["domain-events-and-delius"]'
+          - '["dps-and-delius"]'
           - '["effective-proposal-framework-and-delius"]'
           - '["external-api-and-delius"]'
+          - '["hdc-licences-and-delius"]'
+          - '["hmpps-auth-and-delius"]'
           - '["make-recall-decisions-and-delius"]'
+          - '["manage-offences-and-delius"]'
           - '["manage-pom-cases-and-delius"]'
           - '["offender-events-and-delius"]'
+          - '["opd-and-delius"]'
           - '["pathfinder-and-delius"]'
           - '["person-search-index-from-delius"]'
           - '["pre-sentence-reports-to-delius"]'
           - '["prison-case-notes-to-probation"]'
           - '["prison-custody-status-to-delius"]'
+          - '["prison-education-and-delius"]'
+          - '["prison-identifier-and-delius"]'
+          - '["prisoner-profile-and-delius"]'
           - '["refer-and-monitor-and-delius"]'
+          - '["resettlement-passport-and-delius"]'
           - '["risk-assessment-scores-to-delius"]'
           - '["sentence-plan-and-delius"]'
           - '["sentence-plan-and-oasys"]'
           - '["soc-and-delius"]'
           - '["tier-to-delius"]'
           - '["unpaid-work-and-delius"]'
           - '["workforce-allocations-to-delius"]'
-          - '["manage-offences-and-delius"]'
-          - '["resettlement-passport-and-delius"]'
-          - '["prison-education-and-delius"]'
-          - '["opd-and-delius"]'
-          - '["hmpps-auth-and-delius"]'
-          - '["dps-and-delius"]'
-          - '["arns-and-delius"]'
-          - '["cas3-and-delius"]'
-          - '["hdc-licences-and-delius"]'
-          - '["prisoner-profile-and-delius"]'
-          - '["prison-identifier-and-delius"]'
-          - '["assessment-summary-and-delius"]'
           # ^ add new projects here
           # GitHub Actions doesn't support dynamic choices, we must add each project here to enable manual deployments
           # See https://github.com/community/community/discussions/11795
@@ -68,7 +68,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Get projects - all
         if: github.event_name != 'workflow_dispatch' || inputs.projects == 'All'
-        run: echo "projects=$(find projects -mindepth 1 -maxdepth 1 -printf "%f\n" | jq --raw-input . | jq --slurp --compact-output .)" | tee -a "$GITHUB_ENV"
+        run: echo "projects=$(cd projects && find . -name tech-docs -exec dirname {} \; | sed 's#^\./##' | jq --raw-input . | jq --slurp --compact-output .)" | tee -a "$GITHUB_ENV"
       - name: Get projects - selected
         if: github.event_name == 'workflow_dispatch' && inputs.projects != 'All'
         run: echo 'projects=${{ inputs.projects }}' | tee -a "$GITHUB_ENV"

projects/redrive-dead-letter-queues/README.md

@@ -0,0 +1,7 @@
+# Redrive Dead Letter Queues
+
+A simple CronJob that redrives SQS messages from DLQs back to the source queue.
+
+## References
+
+* https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-configure-dead-letter-queue-redrive.html

projects/redrive-dead-letter-queues/container/Dockerfile

@@ -0,0 +1,15 @@
+FROM public.ecr.aws/aws-cli/aws-cli:2.15.5
+
+USER root
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+# hadolint ignore=DL3033
+RUN yum update -y && \
+    yum install -y shadow-utils jq && \
+    yum clean all && \
+    useradd --uid 1000 --create-home --system app && \
+    curl -sL https://sentry.io/get-cli/ | bash
+USER 1000
+
+COPY --chown=app /scripts /scripts
+
+ENTRYPOINT ["/scripts/startup.sh"]

projects/redrive-dead-letter-queues/container/scripts/redrive.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -euo pipefail
+eval "$(sentry-cli bash-hook --no-environ)"
+
+queue_urls=$(aws sqs list-queues --queue-name-prefix "probation-integration-$ENVIRONMENT" | jq -r '.QueueUrls[] | select(endswith("-dlq"))')
+
+for queue_url in $queue_urls
+do
+  echo "Redriving $queue_url"
+  queue_arn=$(aws sqs get-queue-attributes --attribute-names QueueArn --queue-url "$queue_url" --query Attributes.QueueArn --output text)
+  aws sqs start-message-move-task --source-arn "$queue_arn"
+done

projects/redrive-dead-letter-queues/container/scripts/startup.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -euo pipefail
+eval "$(sentry-cli bash-hook --no-environ)"
+sentry-cli monitors run -e "$SENTRY_ENVIRONMENT" redrive-dead-letter-queues -- /scripts/redrive.sh

projects/redrive-dead-letter-queues/deploy/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+appVersion: '1.0'
+description: A Helm chart for Kubernetes
+name: redrive-dead-letter-queues
+version: 1.0.0

projects/redrive-dead-letter-queues/deploy/templates/cronjob.yml

@@ -0,0 +1,42 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: dlq-redrive
+spec:
+  schedule: {{ .Values.dlq_redrive.schedule }}
+  concurrencyPolicy: Replace
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: hmpps-probation-integration-services
+          containers:
+            - name: dlq-redrive
+              image: "ghcr.io/ministryofjustice/hmpps-probation-integration-services/redrive-dead-letter-queues:{{ .Values.version }}"
+              resources:
+                requests:
+                  memory: 100Mi
+                  cpu: "0.1"
+                limits:
+                  memory: 500Mi
+                  cpu: "1"
+              env:
+                - name: AWS_REGION
+                  value: eu-west-2
+                - name: ENVIRONMENT
+                  value: {{ .Values.dlq_redrive.environment }}
+                - name: SENTRY_ENVIRONMENT
+                  value: {{ .Values.dlq_redrive.environment }}
+                - name: SENTRY_DSN
+                  valueFrom:
+                    secretKeyRef:
+                      name: redrive-dead-letter-queues-sentry
+                      key: SENTRY_DSN
+                      optional: false
+                - name: SENTRY_AUTH_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: redrive-dead-letter-queues-sentry
+                      key: SENTRY_AUTH_TOKEN
+                      optional: false
+          restartPolicy: Never

projects/redrive-dead-letter-queues/deploy/values-dev.yml

@@ -0,0 +1,2 @@
+dlq_redrive:
+  environment: dev

projects/redrive-dead-letter-queues/deploy/values-preprod.yml

@@ -0,0 +1,2 @@
+dlq_redrive:
+  environment: preprod

projects/redrive-dead-letter-queues/deploy/values-prod.yml

@@ -0,0 +1,2 @@
+dlq_redrive:
+  environment: prod

projects/redrive-dead-letter-queues/deploy/values.yaml

@@ -0,0 +1,2 @@
+dlq_redrive:
+  schedule: 30 5 * * 1 # Every Monday at 05:30 UTC