Add excluded urls option

* refactor from pull request comments to compile regex * add exclude test
bartdag · Sep 2, 2015 · 86a8679 · 86a8679 · jimpriest · Sep 2, 2015
1 parent 8838f8c
commit 86a8679
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/pylinkvalidator/models.py b/pylinkvalidator/models.py
@@ -173,7 +173,7 @@ def should_download(self, url_split):
         url = url_split.geturl()
 
         for exclude_url in self.excluded_urls:
-            if re.search(exclude_url, url):
+            if exclude_url.search(url):
                 return False
 
         for ignored_prefix in self.ignored_prefixes:
@@ -215,7 +215,8 @@ def _parse_config(self):
             self.ignored_prefixes = self.options.ignored_prefixes.split(',')
 
         if self.options.excluded_urls:
-            self.excluded_urls = self.options.excluded_urls.split(',')
+            self.excluded_urls = [re.compile(pattern) for pattern in self.options.excluded_urls.split(',')]
+
 
         if self.options.workers:
             self.worker_size = self.options.workers

diff --git a/pylinkvalidator/tests.py b/pylinkvalidator/tests.py
@@ -331,6 +331,13 @@ def test_run_once(self):
         self.assertEqual(8, len(site.pages))
         self.assertEqual(0, len(site.error_pages))
 
+    def test_exclude(self):
+        site = self._run_crawler_plain(ThreadSiteCrawler, ["--exclude=/sub/"])
+
+        # exclude /sub/ directory = 4 pages linked on the index
+        self.assertEqual(4, len(site.pages))
+        self.assertEqual(0, len(site.error_pages))
+
     def test_depth_0(self):
         site = self._run_crawler_plain(
             ThreadSiteCrawler, ["--depth", "0"], "/depth/root.html")