PI-1736 remove dlqs (#2942)

projects/person-search-index-from-delius/container/config/pipelines.yml

@@ -2,12 +2,8 @@
   path.config: /pipelines/person/logstash-full-load.conf
 - pipeline.id: person-incremental
   path.config: /pipelines/person/logstash-incremental.conf
-- pipeline.id: person-dlq
-  path.config: /pipelines/person/logstash-dlq.conf
 
 - pipeline.id: contact-full-load
   path.config: /pipelines/contact/logstash-full-load.conf
 - pipeline.id: contact-incremental
   path.config: /pipelines/contact/logstash-incremental.conf
-- pipeline.id: contact-dlq
-  path.config: /pipelines/contact/logstash-dlq.conf

projects/person-search-index-from-delius/container/pipelines/contact/logstash-full-load.conf

@@ -33,14 +33,7 @@ filter {
 }
 
 output {
-    if [tags] and [tags][0] {
-        sqs {
-            id => "send-to-sqs-dlq"
-            queue => "${CONTACT_SQS_DLQ_NAME}"
-            endpoint => "${SQS_ENDPOINT:https://sqs.eu-west-2.amazonaws.com}"
-            region => "eu-west-2"
-        }
-    } else {
+    unless [tags] and [tags][0] {
         opensearch {
             id => "index-into-standby"
             hosts => ["${SEARCH_INDEX_HOST}"]

projects/person-search-index-from-delius/container/pipelines/contact/logstash-incremental.conf

@@ -41,14 +41,7 @@ filter {
 }
 
 output {
-    if [tags] and [tags][0] {
-        sqs {
-            id => "send-to-sqs-dlq"
-            queue => "${CONTACT_SQS_DLQ_NAME}"
-            endpoint => "${SQS_ENDPOINT:https://sqs.eu-west-2.amazonaws.com}"
-            region => "eu-west-2"
-        }
-    } else {
+    unless [tags] and [tags][0] {
         opensearch {
             id => "index-into-primary"
             hosts => ["${SEARCH_INDEX_HOST}"]

projects/person-search-index-from-delius/container/pipelines/person/logstash-full-load.conf

@@ -33,14 +33,7 @@ filter {
 }
 
 output {
-    if [tags] and [tags][0] {
-        sqs {
-            id => "send-to-sqs-dlq"
-            queue => "${PERSON_SQS_DLQ_NAME}"
-            endpoint => "${SQS_ENDPOINT:https://sqs.eu-west-2.amazonaws.com}"
-            region => "eu-west-2"
-        }
-    } else {
+    unless [tags] and [tags][0] {
         opensearch {
             id => "index-into-standby"
             hosts => ["${SEARCH_INDEX_HOST}"]

projects/person-search-index-from-delius/container/pipelines/person/logstash-incremental.conf

@@ -41,14 +41,7 @@ filter {
 }
 
 output {
-    if [tags] and [tags][0] {
-        sqs {
-            id => "send-to-sqs-dlq"
-            queue => "${PERSON_SQS_DLQ_NAME}"
-            endpoint => "${SQS_ENDPOINT:https://sqs.eu-west-2.amazonaws.com}"
-            region => "eu-west-2"
-        }
-    } else {
+    unless [tags] and [tags][0] {
         opensearch {
             id => "index-into-primary"
             hosts => ["${SEARCH_INDEX_HOST}"]

projects/person-search-index-from-delius/deploy/templates/contact-reindex-cronjob.yml

@@ -32,7 +32,7 @@ spec:
                 - name: CONTACT_REINDEXING_TIMEOUT
                   value: '172800' # 48 hours
                 - name: PIPELINES_ENABLED
-                  value: contact-full-load,contact-dlq
+                  value: contact-full-load
                 - name: PIPELINE_BATCH_SIZE # Maximum number of in-memory events per worker
                   value: '10000'
                 - name: JDBC_BATCH_SIZE # Maximum number of results to return from DB at a time
@@ -85,10 +85,4 @@ spec:
                       name: person-search-index-from-delius-sentry
                       key: CONTACT_REINDEXING_SENTRY_MONITOR_ID
                       optional: false
-                - name: CONTACT_SQS_DLQ_NAME
-                  valueFrom:
-                    secretKeyRef:
-                      name: person-search-index-from-delius-contact-dlq
-                      key: QUEUE_NAME
-                      optional: false
           restartPolicy: Never

projects/person-search-index-from-delius/deploy/templates/person-reindex-cronjob.yml

@@ -32,7 +32,7 @@ spec:
                 - name: PERSON_REINDEXING_TIMEOUT
                   value: '7200' # 3 hours
                 - name: PIPELINES_ENABLED
-                  value: person-full-load,person-dlq
+                  value: person-full-load
                 - name: PIPELINE_BATCH_SIZE # Maximum number of in-memory events per worker
                   value: '5000'
                 - name: JDBC_BATCH_SIZE # Maximum number of results to return from DB at a time
@@ -85,10 +85,4 @@ spec:
                       name: person-search-index-from-delius-sentry
                       key: PERSON_REINDEXING_SENTRY_MONITOR_ID
                       optional: false
-                - name: PERSON_SQS_DLQ_NAME
-                  valueFrom:
-                    secretKeyRef:
-                      name: person-search-index-from-delius-person-dlq
-                      key: QUEUE_NAME
-                      optional: false
           restartPolicy: Never

projects/person-search-index-from-delius/deploy/values.yaml

@@ -27,7 +27,7 @@ generic-service:
   env:
     CONTACT_INDEX_PREFIX: contact-search
     PERSON_INDEX_PREFIX: person-search
-    PIPELINES_ENABLED: person-incremental,person-dlq,contact-incremental,contact-dlq
+    PIPELINES_ENABLED: person-incremental,contact-incremental
     JDK_JAVA_OPTIONS: -XX:MaxRAMPercentage=75.0
 
   namespace_secrets:
@@ -42,12 +42,8 @@ generic-service:
       SENTRY_DSN: SENTRY_DSN
     person-search-index-from-delius-person-queue:
       PERSON_SQS_QUEUE_NAME: QUEUE_NAME
-    person-search-index-from-delius-person-dlq:
-      PERSON_SQS_DLQ_NAME: QUEUE_NAME
     person-search-index-from-delius-contact-queue:
       CONTACT_SQS_QUEUE_NAME: QUEUE_NAME
-    person-search-index-from-delius-contact-dlq:
-      CONTACT_SQS_DLQ_NAME: QUEUE_NAME
 
 generic-prometheus-alerts:
   targetApplication: person-search-index-from-delius

projects/person-search-index-from-delius/docker-compose.yml

@@ -29,12 +29,10 @@ services:
       PERSON_INDEX_SCHEDULE: "* * * * *"
       PERSON_REINDEXING_TIMEOUT: 600
       PERSON_SQS_QUEUE_NAME: person-queue
-      PERSON_SQS_DLQ_NAME: person-dlq
       CONTACT_INDEX_PREFIX: contact-search
       CONTACT_INDEX_SCHEDULE: "* * * * *"
       CONTACT_REINDEXING_TIMEOUT: 600
       CONTACT_SQS_QUEUE_NAME: contact-queue
-      CONTACT_SQS_DLQ_NAME: contact-dlq
     ports:
       - "9600:9600"
     depends_on:

projects/person-search-index-from-delius/localstack/sqs-setup.sh

@@ -1,9 +1,7 @@
 #!/usr/bin/env bash
 echo creating queues...
 awslocal sqs create-queue --queue-name person-queue
-awslocal sqs create-queue --queue-name person-dlq
 awslocal sqs create-queue --queue-name contact-queue
-awslocal sqs create-queue --queue-name contact-dlq
 
 # Testing:
 # awslocal sqs send-message '{"Message": "{\"offenderId\": 1}"}'