feat: use java Normalizer for titleNorm #1173

ICIJ · Sep 1, 2023 · 571bccf · 571bccf
1 parent a72f855
commit 571bccf
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/...dex/src/main/java/org/icij/datashare/text/indexing/elasticsearch/ElasticsearchSpewer.java b/...dex/src/main/java/org/icij/datashare/text/indexing/elasticsearch/ElasticsearchSpewer.java
@@ -27,6 +27,7 @@
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.text.Normalizer;
 import java.util.*;
 
 import static java.lang.System.currentTimeMillis;
@@ -132,7 +133,7 @@ Map<String, Object> getDocumentMap(TikaDocument document) throws IOException {
         jsonDocument.put("contentLength", Long.valueOf(ofNullable(document.getMetadata().get(CONTENT_LENGTH)).orElse("-1")));
         jsonDocument.put("contentEncoding", ofNullable(document.getMetadata().get(CONTENT_ENCODING)).orElse(DEFAULT_VALUE_UNKNOWN));
         jsonDocument.put("title", ofNullable(getTitle(document.getMetadata())).orElse(DEFAULT_VALUE_UNKNOWN));
-        jsonDocument.put("titleNorm", Optional.of(getTitle(document.getMetadata()).toLowerCase()).orElse(DEFAULT_VALUE_UNKNOWN));
+        jsonDocument.put("titleNorm", ofNullable(normalize(getTitle(document.getMetadata()))).orElse(DEFAULT_VALUE_UNKNOWN));
 
         String content = toString(document.getReader()).trim();
         if (maxContentLength != -1 && content.length() > maxContentLength) {
@@ -174,6 +175,12 @@ public ElasticsearchSpewer withRefresh(WriteRequest.RefreshPolicy refreshPolicy)
         return this;
     }
 
+    public static String normalize(String input) {
+        // Normalize special characters to their ASCII equivalents
+        // and convert to lowercase
+        return Normalizer.normalize(input, Normalizer.Form.NFD).toLowerCase();
+    }
+
     int getMaxContentLength(PropertiesProvider propertiesProvider) {
         return (int) Math.min(HumanReadableSize.parse(propertiesProvider.get("maxContentLength").orElse("-1")), Integer.MAX_VALUE);
     }