#115 Adding caching to additional filters along with unit tests.

Author: jzonthemtn

phileas-core/src/main/java/ai/philterd/phileas/services/FilterPolicyLoader.java

@@ -0,0 +1,96 @@
+/*
+ *     Copyright 2024 Philterd, LLC @ https://www.philterd.ai
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.philterd.test.phileas.services;
+
+import ai.philterd.phileas.model.configuration.PhileasConfiguration;
+import ai.philterd.phileas.model.enums.FilterType;
+import ai.philterd.phileas.model.filter.Filter;
+import ai.philterd.phileas.model.policy.Identifiers;
+import ai.philterd.phileas.model.policy.Policy;
+import ai.philterd.phileas.model.policy.filters.ZipCode;
+import ai.philterd.phileas.model.services.AlertService;
+import ai.philterd.phileas.model.services.AnonymizationCacheService;
+import ai.philterd.phileas.model.services.MetricsService;
+import ai.philterd.phileas.services.FilterPolicyLoader;
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class FilterPolicyLoaderTest {
+
+    private static final Logger LOGGER = LogManager.getLogger(FilterPolicyLoaderTest.class);
+
+    @Test
+    public void getFiltersForPolicy() throws Exception {
+
+        final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class);
+        final AlertService alertService = Mockito.mock(AlertService.class);
+        final MetricsService metricsService = Mockito.mock(MetricsService.class);
+        final Map<String, DescriptiveStatistics> stats = new HashMap<>();
+        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(new Properties());
+
+        final FilterPolicyLoader filterPolicyLoader = new FilterPolicyLoader(alertService, anonymizationCacheService, metricsService, stats, phileasConfiguration);
+
+        final Identifiers identifiers = new Identifiers();
+        identifiers.setZipCode(new ZipCode());
+
+        final Policy policy = new Policy();
+        policy.setName("unnamed");
+        policy.setIdentifiers(identifiers);
+
+        final Map<String, Map<FilterType, Filter>> filterCache = new HashMap<>();
+
+        final List<Filter> filters = filterPolicyLoader.getFiltersForPolicy(policy, filterCache);
+
+        Assertions.assertEquals(1, filters.size());
+        Assertions.assertEquals(1, filterCache.size());
+        Assertions.assertNotNull(filterCache.get("unnamed").get(FilterType.ZIP_CODE));
+
+    }
+
+    @Test
+    public void getFiltersForPolicyWithNoFilters() throws Exception {
+
+        final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class);
+        final AlertService alertService = Mockito.mock(AlertService.class);
+        final MetricsService metricsService = Mockito.mock(MetricsService.class);
+        final Map<String, DescriptiveStatistics> stats = new HashMap<>();
+        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(new Properties());
+
+        final FilterPolicyLoader filterPolicyLoader = new FilterPolicyLoader(alertService, anonymizationCacheService, metricsService, stats, phileasConfiguration);
+
+        final Policy policy = new Policy();
+        policy.setName("unnamed");
+
+        final Map<String, Map<FilterType, Filter>> filterCache = new HashMap<>();
+
+        final List<Filter> filters = filterPolicyLoader.getFiltersForPolicy(policy, filterCache);
+
+        Assertions.assertEquals(0, filters.size());
+        Assertions.assertEquals(1, filterCache.size());
+        Assertions.assertEquals(0, filterCache.get("unnamed").size());
+
+    }
+
+}

phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java

@@ -16,16 +16,15 @@
 package ai.philterd.test.phileas.services;
 
 import ai.philterd.phileas.model.configuration.PhileasConfiguration;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
 import ai.philterd.phileas.model.enums.MimeType;
 import ai.philterd.phileas.model.objects.Span;
-import ai.philterd.phileas.model.policy.Policy;
 import ai.philterd.phileas.model.policy.Ignored;
+import ai.philterd.phileas.model.policy.Policy;
 import ai.philterd.phileas.model.responses.BinaryDocumentFilterResponse;
 import ai.philterd.phileas.model.serializers.PlaceholderDeserializer;
 import ai.philterd.phileas.services.PhileasFilterService;
-
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
@@ -47,8 +46,8 @@
 import java.util.List;
 import java.util.Properties;
 
-import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPolicy;
 import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPdfPolicy;
+import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPolicy;
 
 public class PhileasFilterServiceTest {
 
@@ -117,7 +116,7 @@ public void pdf1() throws Exception {
         properties.setProperty("indexes.directory", INDEXES_DIRECTORY);
         properties.setProperty("filter.policies.directory", temp.toFile().getAbsolutePath());
 
-        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties, "phileas");
+        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties);
 
         PhileasFilterService service = new PhileasFilterService(phileasConfiguration);
         final BinaryDocumentFilterResponse response = service.filter(Arrays.asList("pdf"), "context", "documentid", document, MimeType.APPLICATION_PDF, MimeType.APPLICATION_PDF);
@@ -153,7 +152,7 @@ public void pdf2() throws Exception {
         properties.setProperty("indexes.directory", INDEXES_DIRECTORY);
         properties.setProperty("filter.policies.directory", temp.toFile().getAbsolutePath());
 
-        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties, "phileas");
+        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties);
 
         PhileasFilterService service = new PhileasFilterService(phileasConfiguration);
         final BinaryDocumentFilterResponse response = service.filter(Arrays.asList("pdf"), "context", "documentid", document, MimeType.APPLICATION_PDF, MimeType.APPLICATION_PDF);

phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTests.java

@@ -94,7 +94,7 @@ private PhileasConfiguration getConfiguration() throws IOException {
 
         }
 
-        return new PhileasConfiguration(properties, "phileas");
+        return new PhileasConfiguration(properties);
 
     }
 

phileas-core/src/test/java/ai/philterd/test/phileas/services/FilterPolicyLoaderTest.java

@@ -30,6 +30,13 @@ public PhileasConfiguration(final Properties properties, final String applicatio
 
     }
 
+    public PhileasConfiguration(final Properties properties) throws IOException {
+
+        this.properties = properties;
+        this.applicationName = "phileas";
+
+    }
+
     public String indexesDirectory() {
         return getProperty("indexes.directory", "./indexes/");
     }

phileas-core/src/test/java/ai/philterd/test/phileas/services/PhileasFilterServiceTest.java

@@ -27,7 +27,7 @@ public class Policy {
 
     @SerializedName("name")
     @Expose
-    private String name;
+    private String name = "unnamed";
 
     @SerializedName("config")
     @Expose
@@ -51,11 +51,11 @@ public class Policy {
 
     @SerializedName("ignored")
     @Expose
-    private List<Ignored> ignored = Collections.EMPTY_LIST;
+    private List<Ignored> ignored = Collections.emptyList();
 
     @SerializedName("ignoredPatterns")
     @Expose
-    private List<IgnoredPattern> ignoredPatterns = Collections.EMPTY_LIST;
+    private List<IgnoredPattern> ignoredPatterns = Collections.emptyList();
 
     @SerializedName("graphical")
     @Expose

phileas-core/src/test/java/ai/philterd/test/phileas/services/registry/S3PolicyServiceTest.java

@@ -81,7 +81,7 @@ private PhileasConfiguration getConfiguration() throws IOException {
 
         }
 
-        return new PhileasConfiguration(properties, "phileas");
+        return new PhileasConfiguration(properties);
 
     }
 

phileas-model/src/main/java/ai/philterd/phileas/model/configuration/PhileasConfiguration.java

@@ -40,7 +40,7 @@ private PhileasConfiguration getConfiguration() throws IOException {
 
         final Properties properties = new Properties();
 
-        return new PhileasConfiguration(properties, "phileas");
+        return new PhileasConfiguration(properties);
 
     }
 

phileas-model/src/main/java/ai/philterd/phileas/model/policy/Policy.java

@@ -73,7 +73,7 @@ private PhileasConfiguration getConfiguration() throws IOException {
 
         }
 
-        return new PhileasConfiguration(properties, "phileas");
+        return new PhileasConfiguration(properties);
 
     }
 

phileas-services/phileas-services-alerts/src/test/java/ai/philterd/test/phileas/services/alerts/RedisAlertServiceTest.java

@@ -43,7 +43,7 @@ public void disambiguateLocal1() throws IOException {
         properties.setProperty("span.disambiguation.vector.size", "32");
         properties.setProperty("cache.redis.enabled", "false");
 
-        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties, "phileas");
+        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties);
 
         final String context = "c";
 
@@ -77,7 +77,7 @@ public void disambiguateLocal2() throws IOException {
         properties.setProperty("span.disambiguation.vector.size", "32");
         properties.setProperty("cache.redis.enabled", "false");
 
-        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties, "phileas");
+        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties);
 
         final String context = "c";
 

phileas-services/phileas-services-anonymization/src/test/java/ai/philterd/test/phileas/services/anonymization/cache/AnonymizationCacheServiceFactoryTest.java

@@ -56,7 +56,7 @@ public void after() {
     public void disambiguateWithRedis1() throws IOException {
 
         final Properties properties = getProperties();
-        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties, "phileas");
+        final PhileasConfiguration phileasConfiguration = new PhileasConfiguration(properties);
 
         final String context = "c";