Fixed "caseSensitive" flag sometimes having no effect in

RegexMetadataFilter and RegexReferenceFilter.
Norconex · May 26, 2017 · cfe026a · cfe026a
1 parent 30cbeb2
commit cfe026a
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 25 deletions.
diff --git a/norconex-collector-core/pom.xml b/norconex-collector-core/pom.xml
@@ -18,7 +18,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.norconex.collectors</groupId>
   <artifactId>norconex-collector-core</artifactId>
-  <version>1.8.1</version>
+  <version>1.8.2-SNAPSHOT</version>
   <name>Norconex Collector Core</name>
 
   <properties>
@@ -65,7 +65,7 @@
     <dependency>
       <groupId>com.norconex.collectors</groupId>
       <artifactId>norconex-importer</artifactId>
-      <version>2.7.1</version>
+      <version>2.7.2-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.norconex.collectors</groupId>

diff --git a/norconex-collector-core/src/changes/changes.xml b/norconex-collector-core/src/changes/changes.xml
@@ -7,6 +7,16 @@
   </properties>
   <body>
 
+    <release version="1.8.2-SNAPSHOT" date="2017-??-??" description="Bugfix release">
+      <action dev="danizen" type="update">
+        Dependency updates: Norconex Importer 2.7.2.
+      </action>
+      <action dev="essiembre" type="fix">
+        Fixed "caseSensitive" flag sometimes having no effect in 
+        RegexMetadataFilter and RegexReferenceFilter.
+      </action>
+    </release>
+
     <release version="1.8.1" date="2017-05-25" description="Maintenance release">
       <action dev="essiembre,danizen" type="add">
         MongoCrawlDataStore now support specifying the MongoDB authentication 

diff --git a/...-core/src/main/java/com/norconex/collector/core/filter/impl/ExtensionReferenceFilter.java b/...-core/src/main/java/com/norconex/collector/core/filter/impl/ExtensionReferenceFilter.java
@@ -140,8 +140,9 @@ public final void setExtensions(String extensions) {
         this.extensions = extensions;
         if (extensions != null) {
             this.extensionParts = extensions.split("\\s*,\\s*");
-            for (int i = 0; i < this.extensionParts.length; i++)
+            for (int i = 0; i < this.extensionParts.length; i++) {
                 this.extensionParts[i] = this.extensionParts[i].trim();
+            }
         } else {
             this.extensionParts = ArrayUtils.EMPTY_STRING_ARRAY;
         }

diff --git a/...ector-core/src/main/java/com/norconex/collector/core/filter/impl/RegexMetadataFilter.java b/...ector-core/src/main/java/com/norconex/collector/core/filter/impl/RegexMetadataFilter.java
@@ -78,7 +78,7 @@ public class RegexMetadataFilter extends AbstractOnMatchFilter
     private boolean caseSensitive;
     private String field;
     private String regex;
-    private Pattern pattern;
+    private Pattern cachedPattern;
 
     public RegexMetadataFilter() {
         this(null, null, OnMatch.INCLUDE);
@@ -110,21 +110,14 @@ public String getField() {
     }
     public final void setCaseSensitive(boolean caseSensitive) {
         this.caseSensitive = caseSensitive;
+        cachedPattern = null;
     }
     public final void setField(String header) {
         this.field = header;
     }
     public final void setRegex(String regex) {
         this.regex = regex;
-        if (regex != null) {
-            int flags = Pattern.DOTALL;
-            if (!caseSensitive) {
-                flags = flags | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE;
-            }
-            this.pattern = Pattern.compile(regex, flags);            
-        } else {
-            this.pattern = Pattern.compile(".*");
-        }
+        cachedPattern = null;
     }
 
     @Override
@@ -135,13 +128,31 @@ public boolean acceptMetadata(String reference, Properties metadata) {
         Collection<String> values = metadata.getStrings(field);
         for (Object value : values) {
             String strVal = Objects.toString(value, StringUtils.EMPTY);
-            if (pattern.matcher(strVal).matches()) {
+            if (getCachedPattern().matcher(strVal).matches()) {
                 return getOnMatch() == OnMatch.INCLUDE;
             }
         }
         return getOnMatch() == OnMatch.EXCLUDE;
     }
 
+    private synchronized Pattern getCachedPattern() {
+        if (cachedPattern != null) {
+            return cachedPattern;
+        }
+        Pattern p;
+        if (regex == null) {
+            p = Pattern.compile(".*");
+        } else {
+            int flags = Pattern.DOTALL;
+            if (!caseSensitive) {
+                flags = flags | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE;
+            }
+            p = Pattern.compile(regex, flags);
+        }
+        cachedPattern = p;
+        return p;
+    }    
+
     @Override
     public boolean acceptDocument(ImporterDocument document) {
         if (document == null) {

diff --git a/...ctor-core/src/main/java/com/norconex/collector/core/filter/impl/RegexReferenceFilter.java b/...ctor-core/src/main/java/com/norconex/collector/core/filter/impl/RegexReferenceFilter.java
@@ -73,7 +73,7 @@ public class RegexReferenceFilter extends AbstractOnMatchFilter implements
 
     private boolean caseSensitive;
     private String regex;
-    private Pattern pattern;
+    private Pattern cachedPattern;
 
     public RegexReferenceFilter() {
         this(null, OnMatch.INCLUDE);
@@ -103,18 +103,11 @@ public boolean isCaseSensitive() {
     }
     public final void setCaseSensitive(boolean caseSensitive) {
         this.caseSensitive = caseSensitive;
+        cachedPattern = null;
     }
     public final void setRegex(String regex) {
         this.regex = regex;
-        if (regex != null) {
-            int flags = Pattern.DOTALL;
-            if (!caseSensitive) {
-                flags = flags | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE;
-            }
-            this.pattern = Pattern.compile(regex, flags);
-        } else {
-            this.pattern = Pattern.compile(".*");
-        }
+        cachedPattern = null;
     }
 
     @Override
@@ -123,9 +116,27 @@ public boolean acceptReference(String url) {
         if (StringUtils.isBlank(regex)) {
             return isInclude;
         }
-        boolean matches = pattern.matcher(url).matches();
+        boolean matches = getCachedPattern().matcher(url).matches();
         return matches && isInclude || !matches && !isInclude;
     }
+
+    private synchronized Pattern getCachedPattern() {
+        if (cachedPattern != null) {
+            return cachedPattern;
+        }
+        Pattern p;
+        if (regex == null) {
+            p = Pattern.compile(".*");
+        } else {
+            int flags = Pattern.DOTALL;
+            if (!caseSensitive) {
+                flags = flags | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE;
+            }
+            p = Pattern.compile(regex, flags);
+        }
+        cachedPattern = p;
+        return p;
+    }
 
     @Override
     public void loadFromXML(Reader in) {

diff --git a/...-core/src/test/java/com/norconex/collector/core/filter/impl/RegexReferenceFilterTest.java b/...-core/src/test/java/com/norconex/collector/core/filter/impl/RegexReferenceFilterTest.java
@@ -14,6 +14,9 @@
  */
 package com.norconex.collector.core.filter.impl;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
 
 import org.junit.Test;
@@ -22,6 +25,24 @@
 import com.norconex.importer.handler.filter.OnMatch;
 
 public class RegexReferenceFilterTest {
+
+    @Test
+    public void testCaseSensitivity() throws IOException {
+        RegexReferenceFilter f = new RegexReferenceFilter();
+        f.setOnMatch(OnMatch.INCLUDE);
+        f.setRegex("case");
+
+        // must match any case:
+        f.setCaseSensitive(false);
+        assertTrue(f.acceptReference("case"));
+        assertTrue(f.acceptReference("CASE"));
+
+        // must match only matching case:
+        f.setCaseSensitive(true);
+        assertTrue(f.acceptReference("case"));
+        assertFalse(f.acceptReference("CASE"));
+    }
+
 
     @Test
     public void testWriteRead() throws IOException {