PI-2593 Switch to prepared statements for contact indexing

This improves performance by allowing the DB to prepare and cache a single execution plan for the query.
ministryofjustice · Oct 16, 2024 · 677081f · 677081f
1 parent 91159b9
commit 677081f
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 6 deletions.
diff --git a/projects/person-search-index-from-delius/container/Dockerfile b/projects/person-search-index-from-delius/container/Dockerfile
@@ -30,6 +30,9 @@ RUN mkdir -p /usr/share/logstash/data/dead_letter_queue/person-incremental \
 # Workaround for the jdbc_streaming plugin not supporting statement_filepath.
 # See https://github.com/logstash-plugins/logstash-integration-jdbc/issues/51
 RUN sed -i "s/\${INCREMENTAL_STATEMENT_SQL}/$(tr '\n' ' ' < /pipelines/person/statement.sql | sed 's/"/\\\\"/g;s/:batch_size/0/g;s/:sql_last_value/0/g')/" /pipelines/person/logstash-incremental.conf \
- && sed -i "s/\${INCREMENTAL_STATEMENT_SQL}/$(tr '\n' ' ' < /pipelines/contact/statement.sql | sed 's/"/\\\\"/g;s/:batch_size/0/g;s/:sql_last_value/0/g')/" /pipelines/contact/logstash-incremental.conf
+ && sed -i "s/\${INCREMENTAL_STATEMENT_SQL}/$(tr '\n' ' ' < /pipelines/contact/statement.sql | sed 's/"/\\\\"/g;s/:batch_size/0/g;s/:sql_last_value/0/g;/:contact_id/?/g')/" /pipelines/contact/logstash-incremental.conf
+
+# Replace literals with bind parameters
+RUN sed -i -E 's/\B:\w+/?/g' /pipelines/contact/statement.sql
 
 ENTRYPOINT ["/scripts/startup.sh"]
diff --git a/projects/person-search-index-from-delius/container/pipelines/contact/logstash-full-load.conf b/projects/person-search-index-from-delius/container/pipelines/contact/logstash-full-load.conf
@@ -10,10 +10,24 @@ input {
         sql_log_level => "debug"
         lowercase_column_names => false
         statement_filepath => "/pipelines/contact/statement.sql"
-        parameters => {
-            contact_id => 0
-            batch_size => "${JDBC_BATCH_SIZE}"
-        }
+        use_prepared_statements => true
+        prepared_statement_name => "search_indexer_contact_full_load"
+        prepared_statement_bind_values => [
+            ":sql_last_value",
+            "${JDBC_BATCH_SIZE}",
+            ":sql_last_value",
+            "${JDBC_BATCH_SIZE}",
+            0,
+            ":sql_last_value",
+            ":sql_last_value",
+            "${JDBC_BATCH_SIZE}",
+            "${JDBC_BATCH_SIZE}",
+            ":sql_last_value",
+            0,
+            0,
+            ":sql_last_value",
+            0
+        ]
         use_column_value => true
         tracking_column => "sql_next_value"
         tracking_column_type => "numeric"

diff --git a/...cts/person-search-index-from-delius/container/pipelines/contact/logstash-incremental.conf b/...cts/person-search-index-from-delius/container/pipelines/contact/logstash-incremental.conf
@@ -33,7 +33,9 @@ filter {
         jdbc_user => "${JDBC_USER}"
         jdbc_password => "${JDBC_PASSWORD}"
         statement => "${INCREMENTAL_STATEMENT_SQL}"
-        parameters => { "contact_id" => "%{sourceId}" }
+        use_prepared_statements => true
+        prepared_statement_name => "search_indexer_contact_incremental"
+        prepared_statement_bind_values => ["%{sourceId}", "%{sourceId}", "%{sourceId}", "%{sourceId}"]
         target => "db"
         tag_on_default_use => []
     }