Add option to filter duplicate results and deprecate --remove-extensions

CHANGELOG.md

@@ -8,7 +8,7 @@
 - Load targets from a Nmap XML report
 - Added --async option to enable asynchronous mode (use coroutines instead of threads)
 - Added option to disable CLI output entirely
-- Added OpenAI and Ollama REST API endpoints to the dictionary
+- Option to detect and filter identical results
 
 ## [0.4.3] - October 2nd, 2022
 - Automatically detect the URI scheme (`http` or `https`) if no scheme is provided

config.ini

@@ -15,6 +15,7 @@ random-user-agents = False
 max-time = 0
 exit-on-error = False
 skip-on-status = 429
+#filter-threshold = 10
 #subdirs = /,api/
 #include-status = 200-299,401
 #exclude-status = 400,500-999

lib/core/data.py

@@ -35,7 +35,6 @@
     "force_extensions": False,
     "overwrite_extensions": False,
     "exclude_extensions": (),
-    "remove_extensions": None,
     "prefixes": (),
     "suffixes": (),
     "uppercase": False,
@@ -47,6 +46,7 @@
     "force_recursive": False,
     "recursion_depth": 0,
     "recursion_status_codes": set(),
+    "filter_threshold": 0,
     "subdirs": [],
     "exclude_subdirs": [],
     "include_status_codes": set(),

lib/core/dictionary.py

@@ -122,9 +122,6 @@ def generate(self, files: list[str] = [], is_blacklist: bool = False) -> list[st
                 # Removing leading "/" to work with prefixes later
                 line = lstrip_once(line, "/")
 
-                if options["remove_extensions"]:
-                    line = line.split(".")[0]
-
                 if not self.is_valid(line):
                     continue
 

lib/core/fuzzer.py

@@ -53,6 +53,7 @@ def __init__(
         self._requester = requester
         self._dictionary = dictionary
         self._base_path: str = ""
+        self._hashes: dict = {}
         self.match_callbacks = match_callbacks
         self.not_found_callbacks = not_found_callbacks
         self.error_callbacks = error_callbacks
@@ -81,8 +82,7 @@ def get_scanners_for(self, path: str) -> Generator[BaseScanner, None, None]:
         for scanner in self.scanners["default"].values():
             yield scanner
 
-    @staticmethod
-    def is_excluded(resp: BaseResponse) -> bool:
+    def is_excluded(self, resp: BaseResponse) -> bool:
         """Validate the response by different filters"""
 
         if resp.status in options["exclude_status_codes"]:
@@ -127,6 +127,12 @@ def is_excluded(resp: BaseResponse) -> bool:
         ):
             return True
 
+        if (
+            options["filter_threshold"]
+            and self._hashes.get(hash(resp), 0) >= options["filter_threshold"]
+        ):
+            return True
+
         return False
 
 
@@ -252,6 +258,11 @@ def scan(self, path: str) -> None:
                     callback(response)
                 return
 
+        if options["filter_threshold"]:
+            hash_ = hash(response)
+            self._hashes.setdefault(hash_, 0)
+            self._hashes[hash_] += 1
+
         for callback in self.match_callbacks:
             callback(response)
 
@@ -390,6 +401,11 @@ async def scan(self, path: str) -> None:
                     callback(response)
                 return
 
+        if options["filter_threshold"]:
+            hash_ = hash(response)
+            self._hashes.setdefault(hash_, 0)
+            self._hashes[hash_] += 1
+
         for callback in self.match_callbacks:
             callback(response)
 

lib/core/options.py

@@ -70,7 +70,7 @@ def parse_options() -> dict[str, Any]:
             )
         )
 
-    if not opt.extensions and not opt.remove_extensions:
+    if not opt.extensions:
         print("WARNING: No extension was specified!")
 
     if not opt.wordlists:
@@ -151,9 +151,7 @@ def parse_options() -> dict[str, Any]:
     ]
     opt.exclude_sizes = {size.strip().upper() for size in opt.exclude_sizes.split(",")}
 
-    if opt.remove_extensions:
-        opt.extensions = ("",)
-    elif opt.extensions == "*":
+    if opt.extensions == "*":
         opt.extensions = COMMON_EXTENSIONS
     elif opt.extensions == "CHANGELOG.md":
         print(
@@ -271,6 +269,7 @@ def merge_config(opt: Values) -> Values:
     # General
     opt.thread_count = opt.thread_count or config.safe_getint("general", "threads", 25)
     opt.async_mode = opt.async_mode or config.safe_getboolean("general", "async")
+    opt.filter_threshold = opt.filter_threshold or config.safe_getint("general", "filter-threshold", 0)
     opt.include_status_codes = opt.include_status_codes or config.safe_get(
         "general", "include-status"
     )

lib/parse/cmdline.py

@@ -116,12 +116,6 @@ def parse_arguments() -> Values:
         metavar="EXTENSIONS",
         help="Exclude extension list, separated by commas (e.g. asp,jsp)",
     )
-    dictionary.add_option(
-        "--remove-extensions",
-        action="store_true",
-        dest="remove_extensions",
-        help="Remove extensions in all paths (e.g. admin.php -> admin)",
-    )
     dictionary.add_option(
         "--prefixes",
         action="store",
@@ -209,6 +203,14 @@ def parse_arguments() -> Values:
         metavar="CODES",
         help="Valid status codes to perform recursive scan, support ranges (separated by commas)",
     )
+    general.add_option(
+        "--filter-threshold",
+        action="store",
+        type="int",
+        dest="filter_threshold",
+        metavar="THRESHOLD",
+        help="Maximum number of results with duplicate responses before getting filtered out",
+    )
     general.add_option(
         "--subdirs",
         action="store",