#126 Setting up applied and identified spans.

Author: jzonthemtn

phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTests.java

@@ -575,19 +575,23 @@ public void endToEndJustPhoneNumbers() throws Exception {
 
         final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties);
 
-        final String input = "his number is 123-456-7890. her number is 9999999999.";
+        final String input = "his number is 123-456-7890. her number is 9999999999. her number is 102-304-5678.";
 
         final PhileasFilterService service = new PhileasFilterService(phileasConfiguration);
         final FilterResponse response = service.filter(List.of("phonenumbers"), "context", "documentid", input, MimeType.TEXT_PLAIN);
 
         LOGGER.info(response.filteredText());
 
+        LOGGER.info("Identified spans:");
+        showSpans(response.explanation().identifiedSpans());
+
+        LOGGER.info("Applied spans:");
         showSpans(response.explanation().appliedSpans());
 
         Assertions.assertEquals("documentid", response.documentId());
-        Assertions.assertEquals(2, response.explanation().identifiedSpans().size());
         Assertions.assertEquals(1, response.explanation().appliedSpans().size());
-        Assertions.assertEquals("his number is {{{REDACTED-phone-number}}}. her number is {{{REDACTED-phone-number}}}.", response.filteredText().trim());
+        Assertions.assertEquals(3, response.explanation().identifiedSpans().size());
+        Assertions.assertEquals("his number is {{{REDACTED-phone-number}}}. her number is 9999999999. her number is 102-304-5678.", response.filteredText().trim());
 
     }
 

phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTestsHelper.java

@@ -394,6 +394,7 @@ public static Policy getPolicyJustPhoneNumber(String policyName) {
 
         PhoneNumber phoneNumber = new PhoneNumber();
         phoneNumber.setPhoneNumberFilterStrategies(List.of(phoneNumberFilterStrategy));
+        phoneNumber.setIgnored(Set.of("102-304-5678"));
 
         Identifiers identifiers = new Identifiers();
         identifiers.setPhoneNumber(phoneNumber);

phileas-model/src/main/java/ai/philterd/phileas/model/objects/Span.java

@@ -312,11 +312,11 @@ public static List<Span> getIdenticalSpans(Span span, List<Span> spans) {
     }
 
     /**
-     * Drop overlapping spans that were for text that was ignored.
+     * Drop unapplied spans and spans that were for text that was ignored.
      * @param spans A list of {@link Span spans} that may or may not contain ignored spans.
      * @return A list of {@link Span spans} without ignored spans.
      */
-    public static List<Span> dropIgnoredSpans(List<Span> spans) {
+    public static List<Span> dropIgnoredAndUnappliedSpans(List<Span> spans) {
 
         final List<Span> nonIgnoredSpans = new LinkedList<>();
 

phileas-model/src/main/java/ai/philterd/phileas/model/responses/FilterResponse.java

@@ -20,7 +20,6 @@
 import com.google.gson.Gson;
 
 import java.util.*;
-import java.util.stream.Collectors;
 
 /**
  * Response to a filter operation.

phileas-model/src/test/java/ai/philterd/test/phileas/model/objects/SpanTest.java

@@ -132,7 +132,7 @@ public void ignored1() {
         spans.add(Span.make(1, 5, FilterType.PERSON, "context", "document", 1.0, "test", "***", "salt",  false, true, new String[0]));
         spans.add(Span.make(2, 12, FilterType.PERSON, "context", "document", 1.0, "test", "***", "salt",  true, true, new String[0]));
 
-        List<Span> nonIgnoredSpans = Span.dropIgnoredSpans(spans);
+        List<Span> nonIgnoredSpans = Span.dropIgnoredAndUnappliedSpans(spans);
 
         showSpans(nonIgnoredSpans);
 
@@ -148,7 +148,7 @@ public void ignored2() {
         spans.add(Span.make(1, 5, FilterType.PERSON, "context", "document", 1.0, "test", "***", "salt",  false, true, new String[0]));
         spans.add(Span.make(2, 12, FilterType.PERSON, "context", "document", 1.0, "test", "***", "salt",  false, true, new String[0]));
 
-        List<Span> nonIgnoredSpans = Span.dropIgnoredSpans(spans);
+        List<Span> nonIgnoredSpans = Span.dropIgnoredAndUnappliedSpans(spans);
 
         showSpans(nonIgnoredSpans);
 
@@ -165,7 +165,7 @@ public void ignored3() {
         spans.add(Span.make(1, 5, FilterType.PERSON, "context", "document", 1.0, "test", "***", "salt",  true, true, new String[0]));
         spans.add(Span.make(2, 12, FilterType.PERSON, "context", "document", 1.0, "test", "***", "salt",  true, true, new String[0]));
 
-        List<Span> nonIgnoredSpans = Span.dropIgnoredSpans(spans);
+        List<Span> nonIgnoredSpans = Span.dropIgnoredAndUnappliedSpans(spans);
 
         showSpans(nonIgnoredSpans);
 

phileas-processors/phileas-processors-unstructured/src/main/java/ai/philterd/phileas/processors/unstructured/UnstructuredDocumentProcessor.java

@@ -30,6 +30,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Predicate;
 
 import static java.util.stream.Collectors.toList;
 
@@ -55,7 +56,7 @@ public FilterResponse process(final Policy policy, final List<Filter> filters, f
                                   final Map<String, String> attributes) throws Exception {
 
         // The list that will contain the spans containing PHI/PII.
-        List<Span> spans = new LinkedList<>();
+        List<Span> identifiedSpans = new LinkedList<>();
 
         // Apply each filter.
         for(final Filter filter : filters) {
@@ -66,27 +67,27 @@ public FilterResponse process(final Policy policy, final List<Filter> filters, f
 
             metricsService.logFilterTime(filter.getFilterType(), elapsedTimeMs);
 
-            spans.addAll(filterResult.getSpans());
+            identifiedSpans.addAll(filterResult.getSpans());
 
         }
 
         // Drop ignored spans.
-        spans = Span.dropIgnoredSpans(spans);
+        //identifiedSpans = Span.dropIgnoredAndUnappliedSpans(identifiedSpans);
 
         // Perform span disambiguation.
         if(spanDisambiguationService.isEnabled()) {
-            spans = spanDisambiguationService.disambiguate(context, spans);
+            identifiedSpans = spanDisambiguationService.disambiguate(context, identifiedSpans);
         }
 
         // Drop overlapping spans.
-        spans = Span.dropOverlappingSpans(spans);
+        identifiedSpans = Span.dropOverlappingSpans(identifiedSpans);
 
         // Sort the spans based on the confidence.
-        spans.sort(Comparator.comparing(Span::getConfidence));
+        identifiedSpans.sort(Comparator.comparing(Span::getConfidence));
 
         // Perform post-filtering on the spans.
         for(final PostFilter postFilter : postFilters) {
-            spans = postFilter.filter(input, spans);
+            identifiedSpans = postFilter.filter(input, identifiedSpans);
         }
 
         // PHL-185: Remove non-adjacent firstname/surname spans.
@@ -127,15 +128,16 @@ public FilterResponse process(final Policy policy, final List<Filter> filters, f
 
         // The spans that will be persisted. Has to be a deep copy because the shift
         // below will change the indexes. Doing this to save the original locations of the spans.
-        final List<Span> appliedSpans = spans.stream().map(Span::copy).collect(toList());
+        final List<Span> appliedSpans = identifiedSpans.stream().filter(Span::isApplied)
+                .filter(Predicate.not(Span::isIgnored)).map(Span::copy).collect(toList());
 
         // TODO: Set a flag on each "span" not in appliedSpans indicating it was not used.
 
         // Log a metric for each filter type.
         appliedSpans.forEach(k -> metricsService.incrementFilterType(k.getFilterType()));
 
         // Define the explanation.
-        final Explanation explanation = new Explanation(appliedSpans, spans);
+        final Explanation explanation = new Explanation(appliedSpans, identifiedSpans);
 
         // Used to manipulate the text.
         final StringBuilder sb = new StringBuilder(input);
@@ -148,7 +150,7 @@ public FilterResponse process(final Policy policy, final List<Filter> filters, f
         for(int i = 0; i < stringLength; i++) {
 
             // Is index i the start of a span?
-            final Span span = Span.doesIndexStartSpan(i, spans);
+            final Span span = Span.doesIndexStartSpan(i, appliedSpans);
 
             if(span != null) {
 
@@ -166,7 +168,7 @@ public FilterResponse process(final Policy policy, final List<Filter> filters, f
                     final int shift = (spanLength - replacementLength) * -1;
 
                     // Shift the remaining spans by the shift value.
-                    spans = Span.shiftSpans(shift, span, spans);
+                    identifiedSpans = Span.shiftSpans(shift, span, identifiedSpans);
 
                     // Update the length of the string.
                     stringLength += shift;