Merge pull request #54 from ajinabraham/3.1.5

Support multiple multiprocessing options

libsast/__init__.py

@@ -12,7 +12,7 @@
 __title__ = 'libsast'
 __authors__ = 'Ajin Abraham'
 __copyright__ = f'Copyright {year} Ajin Abraham, opensecurity.in'
-__version__ = '3.1.4'
+__version__ = '3.1.5'
 __version_info__ = tuple(int(i) for i in __version__.split('.'))
 __all__ = [
     'Scanner',

libsast/core_matcher/choice_matcher.py

@@ -24,7 +24,7 @@ def __init__(self, options: dict) -> None:
         self.scan_rules = get_rules(options.get('choice_rules'))
         self.show_progress = options.get('show_progress')
         self.cpu = options.get('cpu_core')
-        self.queue = options.get('queue')
+        self.multiprocessing = options.get('multiprocessing')
         self.alternative_path = options.get('alternative_path')
         exts = options.get('choice_extensions')
         self.exts = [ext.lower() for ext in exts] if exts else []
@@ -72,14 +72,20 @@ def regex_scan(self, file_contents: list, rules=None) -> dict:
             return {}
         self.validate_rules()
 
-        if self.queue:
+        if self.multiprocessing == 'billiard':
             # Use billiard's pool for regex (support queues)
             from billiard import Pool
             with Pool(processes=self.cpu) as pool:
                 # Run regex on file data
                 results = pool.map(
                     self.choice_matcher,
                     file_contents)
+        elif self.multiprocessing == 'thread':
+            # Use a ThreadPool for regex check
+            with ThreadPoolExecutor() as io_executor:
+                results = list(io_executor.map(
+                    self.choice_matcher,
+                    file_contents))
         else:
             # Use ProcessPoolExecutor for regex processing
             with ProcessPoolExecutor(max_workers=self.cpu) as cpu_executor:

libsast/core_matcher/pattern_matcher.py

@@ -25,7 +25,7 @@ def __init__(self, options: dict) -> None:
         self.scan_rules = get_rules(options.get('match_rules'))
         self.show_progress = options.get('show_progress')
         self.cpu = options.get('cpu_core')
-        self.queue = options.get('queue')
+        self.multiprocessing = options.get('multiprocessing')
         exts = options.get('match_extensions')
         self.exts = [ext.lower() for ext in exts] if exts else []
         self.findings = {}
@@ -68,7 +68,7 @@ def regex_scan(self, file_contents: list, rules=None) -> dict:
             return {}
         self.validate_rules()
 
-        if self.queue:
+        if self.multiprocessing == 'billiard':
             # Use billiard's pool for CPU-bound regex (support queues)
             from billiard import Pool
             with Pool(processes=self.cpu) as cpu_executor:
@@ -77,6 +77,14 @@ def regex_scan(self, file_contents: list, rules=None) -> dict:
                     self.pattern_matcher,
                     file_contents,
                 )
+        elif self.multiprocessing == 'thread':
+            # Use a ThreadPool for regex check
+            with ThreadPoolExecutor() as io_executor:
+                # Run regex on file data
+                results = io_executor.map(
+                    self.pattern_matcher,
+                    file_contents,
+                )
         else:
             # Use a ProcessPool for CPU-bound regex
             with ProcessPoolExecutor(max_workers=self.cpu) as cpu_executor:

libsast/scanner.py

@@ -26,7 +26,7 @@ def __init__(self, options: dict, paths: list) -> None:
             'ignore_paths': [],
             'show_progress': False,
             'cpu_core': 1,
-            'queue': False,
+            'multiprocessing': 'default',
             # Overwrite with options from invocation
             **(options or {}),
         }

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "libsast"
-version = "3.1.4"
+version = "3.1.5"
 description = "A generic SAST library built on top of semgrep and regex"
 keywords = ["libsast", "SAST", "Python SAST", "SAST API", "Regex SAST", "Pattern Matcher"]
 authors = ["Ajin Abraham <[email protected]>"]