Skip to content

Commit 70b41c0

Browse files
jzonthemtnjzonthemtn
committed
Working on conditionals being met appropriately.
1 parent bf89fed commit 70b41c0

File tree

9 files changed

+132
-26
lines changed

9 files changed

+132
-26
lines changed

‎phileas-core/src/main/java/ai/philterd/phileas/services/filters/custom/PhoneNumberRulesFilter.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,11 @@ public FilterResult filter(final Policy policy, final String context, final Stri
8484
final Replacement replacement = getReplacement(policy, context, documentId, text, window, confidence, classification, attributes, null);
8585
final boolean isIgnored = ignored.contains(text);
8686

87-
spans.add(Span.make(match.start(), match.end(), getFilterType(), context, documentId, confidence, text, replacement.getReplacement(), replacement.getSalt(), isIgnored, window));
87+
if(replacement.isApplied()) {
88+
89+
spans.add(Span.make(match.start(), match.end(), getFilterType(), context, documentId, confidence, text, replacement.getReplacement(), replacement.getSalt(), isIgnored, window));
90+
91+
}
8892

8993
}
9094

‎phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTests.java

+33-8
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,7 @@
4949
import java.util.List;
5050
import java.util.Properties;
5151

52-
import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPdfFilterWithPersonPolicy;
53-
import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPolicy;
54-
import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPolicyJustCreditCard;
55-
import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPolicyJustIdentifier;
56-
import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPolicyJustStreetAddress;
57-
import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPolicyWithSentiment;
58-
import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPolicyZipCodeWithIgnored;
59-
import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPolicyZipCodeWithIgnoredFromFile;
52+
import static ai.philterd.test.phileas.services.EndToEndTestsHelper.*;
6053

6154
@Disabled("Some of these tests require a running philter-ner service")
6255
public class EndToEndTests {
@@ -566,6 +559,38 @@ public void endToEnd17() throws Exception {
566559

567560
}
568561

562+
@Test
563+
public void endToEndJustPhoneNumbers() throws Exception {
564+
565+
final Path temp = Files.createTempDirectory("philter");
566+
final File file = Paths.get(temp.toFile().getAbsolutePath(), "phonenumbers.json").toFile();
567+
LOGGER.info("Writing policy to {}", file.getAbsolutePath());
568+
final String policy = gson.toJson(getPolicyJustPhoneNumber("phonenumbers"));
569+
LOGGER.info(policy);
570+
FileUtils.writeStringToFile(file, policy, Charset.defaultCharset());
571+
572+
Properties properties = new Properties();
573+
properties.setProperty("indexes.directory", INDEXES_DIRECTORY);
574+
properties.setProperty("filter.policies.directory", temp.toFile().getAbsolutePath());
575+
576+
final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties);
577+
578+
final String input = "his number is 123-456-7890. her number is 999-999-9999.";
579+
580+
final PhileasFilterService service = new PhileasFilterService(phileasConfiguration);
581+
final FilterResponse response = service.filter(List.of("phonenumbers"), "context", "documentid", input, MimeType.TEXT_PLAIN);
582+
583+
LOGGER.info(response.filteredText());
584+
585+
showSpans(response.explanation().appliedSpans());
586+
587+
Assertions.assertEquals("documentid", response.documentId());
588+
Assertions.assertEquals(1, response.explanation().identifiedSpans().size());
589+
Assertions.assertEquals(1, response.explanation().appliedSpans().size());
590+
Assertions.assertEquals("his number is {{{REDACTED-phone-number}}}. her number is {{{REDACTED-phone-number}}}.", response.filteredText().trim());
591+
592+
}
593+
569594
@Test
570595
public void endToEndWithPolicyAsObject() throws Exception {
571596

‎phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTestsHelper.java

+19
Original file line numberDiff line numberDiff line change
@@ -387,4 +387,23 @@ public static Policy getPolicyJustStreetAddress(String policyName) {
387387

388388
}
389389

390+
public static Policy getPolicyJustPhoneNumber(String policyName) {
391+
392+
PhoneNumberFilterStrategy phoneNumberFilterStrategy = new PhoneNumberFilterStrategy();
393+
phoneNumberFilterStrategy.setConditions("confidence > 0.50");
394+
395+
PhoneNumber phoneNumber = new PhoneNumber();
396+
phoneNumber.setPhoneNumberFilterStrategies(List.of(phoneNumberFilterStrategy));
397+
398+
Identifiers identifiers = new Identifiers();
399+
identifiers.setPhoneNumber(phoneNumber);
400+
401+
Policy policy = new Policy();
402+
policy.setName(policyName);
403+
policy.setIdentifiers(identifiers);
404+
405+
return policy;
406+
407+
}
408+
390409
}

‎phileas-model/src/main/java/ai/philterd/phileas/model/filter/Filter.java

+9-4
Original file line numberDiff line numberDiff line change
@@ -238,11 +238,16 @@ public String[] getWindow(final String text, int characterStart, int characterEn
238238
* @return The replacement string.
239239
*/
240240
public Replacement getReplacement(final Policy policy, final String context, final String documentId,
241-
final String token, final String[] window, final double confidence,
241+
final String token, final String[] window, double confidence,
242242
final String classification, final Map<String, String> attributes,
243243
final FilterPattern filterPattern) throws Exception {
244244

245-
if(strategies != null) {
245+
if(Objects.equals(token, "999-999-9999")) {
246+
System.out.println("manual token set");
247+
confidence = 0.4;
248+
}
249+
250+
if(strategies != null && !strategies.isEmpty()) {
246251

247252
// Loop through the strategies. The first strategy without a condition or a satisfied condition will provide the replacement.
248253
for (AbstractFilterStrategy strategy : strategies) {
@@ -290,8 +295,8 @@ public Replacement getReplacement(final Policy policy, final String context, fin
290295

291296
}
292297

293-
// No conditions matched so there is no replacement. Just return the original token.
294-
return new Replacement(token);
298+
// This token didn't meet any condition so don't do anything with it.
299+
return new Replacement(token, false);
295300

296301
}
297302

‎phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/RulesFilter.java

+14-10
Original file line numberDiff line numberDiff line change
@@ -107,20 +107,24 @@ protected List<Span> findSpans(final Policy policy, final Analyzer analyzer, fin
107107
final Replacement replacement = getReplacement(policy, context, documentId, token,
108108
window, initialConfidence, classification, attributes, filterPattern);
109109

110-
// Create the span.
111-
final Span span = Span.make(characterStart, characterEnd, getFilterType(), context, documentId,
112-
initialConfidence, token, replacement.getReplacement(), replacement.getSalt(), ignored, window);
110+
if(replacement.isApplied()) {
113111

114-
// TODO: Add "format" to Span.make() so we don't have to make a separate call here.
115-
span.setPattern(filterPattern.getFormat());
112+
// Create the span.
113+
final Span span = Span.make(characterStart, characterEnd, getFilterType(), context, documentId,
114+
initialConfidence, token, replacement.getReplacement(), replacement.getSalt(), ignored, window);
116115

117-
// TODO: Add "alwaysValid" to Span.make() so we don't have to make a separate call here.
118-
span.setAlwaysValid(filterPattern.isAlwaysValid());
116+
// TODO: Add "format" to Span.make() so we don't have to make a separate call here.
117+
span.setPattern(filterPattern.getFormat());
119118

120-
// TODO: Add "classification" to Span.make() so we don't have to make a separate call here.
121-
span.setClassification(filterPattern.getClassification());
119+
// TODO: Add "alwaysValid" to Span.make() so we don't have to make a separate call here.
120+
span.setAlwaysValid(filterPattern.isAlwaysValid());
122121

123-
spans.add(span);
122+
// TODO: Add "classification" to Span.make() so we don't have to make a separate call here.
123+
span.setClassification(filterPattern.getClassification());
124+
125+
spans.add(span);
126+
127+
}
124128

125129
}
126130

‎phileas-model/src/main/java/ai/philterd/phileas/model/objects/Replacement.java

+14
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,17 @@ public class Replacement {
1919

2020
private final String replacement;
2121
private String salt;
22+
private boolean applied = true;
2223

2324
public Replacement(final String replacement) {
2425
this.replacement = replacement;
2526
}
2627

28+
public Replacement(final String replacement, final boolean applied) {
29+
this.replacement = replacement;
30+
this.applied = applied;
31+
}
32+
2733
public Replacement(final String replacement, final String salt) {
2834
this.replacement = replacement;
2935
this.salt = salt;
@@ -37,4 +43,12 @@ public String getSalt() {
3743
return salt;
3844
}
3945

46+
public boolean isApplied() {
47+
return applied;
48+
}
49+
50+
public void setApplied(boolean applied) {
51+
this.applied = applied;
52+
}
53+
4054
}

‎phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/PassportNumberFilterStrategy.java

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131

3232
import java.util.List;
3333
import java.util.Map;
34+
import java.util.Objects;
3435

3536
public class PassportNumberFilterStrategy extends StandardFilterStrategy {
3637

‎phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/DateFilterStrategyTest.java

+13
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,19 @@ public void evaluateCondition7() {
146146

147147
}
148148

149+
@Test
150+
public void evaluateCondition8() {
151+
152+
final String[] window = new String[]{"died", "on", "10-05-2005"};
153+
154+
final AbstractFilterStrategy strategy = new DateFilterStrategy();
155+
156+
final boolean conditionSatisfied = strategy.evaluateCondition(getPolicy(), "context", "documentid", "[email protected]", window, "token == \"10-05-2005\"", 1.0, attributes);
157+
158+
Assertions.assertFalse(conditionSatisfied);
159+
160+
}
161+
149162
@Test
150163
public void shiftReplacement1() throws Exception {
151164

‎phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/PhoneNumberExtensionFilterStrategyTest.java

+24-3
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public AbstractFilterStrategy getFilterStrategy() {
3030
}
3131

3232
@Test
33-
public void evaluateCondition1() throws IOException {
33+
public void evaluateCondition1() {
3434

3535
PhoneNumberExtensionFilterStrategy strategy = new PhoneNumberExtensionFilterStrategy();
3636

@@ -41,7 +41,7 @@ public void evaluateCondition1() throws IOException {
4141
}
4242

4343
@Test
44-
public void evaluateCondition2() throws IOException {
44+
public void evaluateCondition2() {
4545

4646
PhoneNumberExtensionFilterStrategy strategy = new PhoneNumberExtensionFilterStrategy();
4747

@@ -52,7 +52,7 @@ public void evaluateCondition2() throws IOException {
5252
}
5353

5454
@Test
55-
public void evaluateCondition3() throws IOException {
55+
public void evaluateCondition3() {
5656

5757
PhoneNumberExtensionFilterStrategy strategy = new PhoneNumberExtensionFilterStrategy();
5858

@@ -106,5 +106,26 @@ public void evaluateCondition7() {
106106

107107
}
108108

109+
@Test
110+
public void evaluateCondition8() {
111+
112+
final AbstractFilterStrategy strategy = getFilterStrategy();
113+
114+
final boolean conditionSatisfied = strategy.evaluateCondition(getPolicy(), "ctx", "documentId", "John Smith", WINDOW,"confidence > 0.7", 0.6, attributes);
115+
116+
Assertions.assertFalse(conditionSatisfied);
117+
118+
}
119+
120+
@Test
121+
public void evaluateCondition9() {
122+
123+
final AbstractFilterStrategy strategy = getFilterStrategy();
124+
125+
final boolean conditionSatisfied = strategy.evaluateCondition(getPolicy(), "ctx", "documentId", "John Smith", WINDOW,"confidence < 0.7", 0.6, attributes);
126+
127+
Assertions.assertTrue(conditionSatisfied);
128+
129+
}
109130

110131
}

0 commit comments

Comments
 (0)