feat: Added functionality to specify a custom scan when using the sca…

…n command (#565)

secureli-XXX

<!-- Include general description here -->


## Changes
This PR improves the `scan` action by adding new functionality allowing
users to specify a custom scan id instead of only being able to specify
pre-commit hook ids. For example, you can now do `secureli scan -t
check-pii` to run the pii scan.

A new service was introduced, CustomScannersService to help orchestrate
which custom scans should be run. Either a specific scan if an Id is
specified, all custom scans if no id is specified, or a None result is
returned if the specified id doesn't match a value in the new
CustomScanId enum. 

There was also some refactoring done.
modules/core/core_services/scanner.py is now
modules/core/core_services/**hook**_scanner.py to more accurately
describe its function. The pii scanner and custom_regex_scanner
directories have been moved into a new directory;
secureli/modules/custom_scanners/

## Testing
Added unit tests and performed manual testing to confirm that pre-commit
hooks can be specified, custom scans can be specified, and when no id is
specified, then all scans are done

## Clean Code Checklist
<!-- This is here to support you. Some/most checkboxes may not apply to
your change -->
- [x] Meets acceptance criteria for issue
- [x] New logic is covered with automated tests
- [ ] Appropriate exception handling added
- [ ] Thoughtful logging included
- [ ] Documentation is updated
- [ ] Follow-up work is documented in TODOs
- [ ] TODOs have a ticket associated with them
- [x] No commented-out code included


<!--
Github-flavored markdown reference:
https://docs.github.com/en/get-started/writing-on-github
-->

---------

Co-authored-by: Ian Bowden <ian.bowden@slalom>

secureli/actions/action.py

@@ -11,7 +11,7 @@
 from secureli.modules.shared.models.logging import LogAction
 from secureli.modules.shared.models.scan import ScanMode
 from secureli.modules.language_analyzer import language_analyzer, language_support
-from secureli.modules.core.core_services.scanner import HooksScannerService
+from secureli.modules.core.core_services.hook_scanner import HooksScannerService
 from secureli.modules.core.core_services.updater import UpdaterService
 
 from secureli.modules.shared.utilities import format_sentence_list

secureli/actions/scan.py

@@ -4,21 +4,16 @@
 from time import time
 from typing import Optional
 
-from secureli.modules.shared.abstractions.echo import EchoAbstraction
+from secureli.modules.custom_scanners.custom_scans import CustomScannersService
 from secureli.actions import action
 from secureli.modules.shared.abstractions.repo import GitRepo
 from secureli.modules.shared.models.exit_codes import ExitCode
 from secureli.modules.shared.models import install
 from secureli.modules.shared.models.logging import LogAction
 from secureli.modules.shared.models.publish_results import PublishResultsOption
 from secureli.modules.shared.models.result import Result
-from secureli.modules.observability.observability_services.logging import LoggingService
-from secureli.modules.core.core_services.scanner import HooksScannerService
-from secureli.modules.pii_scanner.pii_scanner import PiiScannerService
-from secureli.modules.custom_regex_scanner.custom_regex_scanner import (
-    CustomRegexScannerService,
-)
-from secureli.modules.shared.models.scan import ScanMode, ScanResult
+from secureli.modules.core.core_services.hook_scanner import HooksScannerService
+from secureli.modules.shared.models.scan import ScanMode
 from secureli.settings import Settings
 from secureli.modules.shared import utilities
 
@@ -40,14 +35,12 @@ def __init__(
         self,
         action_deps: action.ActionDependencies,
         hooks_scanner: HooksScannerService,
-        pii_scanner: PiiScannerService,
-        custom_regex_scanner: CustomRegexScannerService,
+        custom_scanners: CustomScannersService,
         git_repo: GitRepo,
     ):
         super().__init__(action_deps)
         self.hooks_scanner = hooks_scanner
-        self.pii_scanner = pii_scanner
-        self.custom_regex_scanner = custom_regex_scanner
+        self.custom_scanners = custom_scanners
         self.git_repo = git_repo
 
     def publish_results(
@@ -92,7 +85,7 @@ def scan_repo(
         :param folder_path: The folder path to initialize the repo for
         :param scan_mode: How we should scan the files in the repo (i.e. staged only or all)
         :param always_yes: Assume "Yes" to all prompts
-        :param specific_test: If set, limits scanning to the single pre-commit hook.
+        :param specific_test: If set, limits scanning to the single pre-commit hook or custom scan.
         :param files: If set, scans only the files provided.
         Otherwise, scans with all hooks.
         """
@@ -119,33 +112,26 @@ def scan_repo(
         if verify_result.outcome in self.halting_outcomes:
             return
 
-        # Execute PII scan (unless `specific_test` is provided, in which case it will be for a hook below)
-        pii_scan_result: ScanResult | None = None
-        custom_regex_patterns = self._get_custom_scan_patterns(folder_path=folder_path)
-        custom_scan_result: ScanResult | None = None
-        if not specific_test:
-            pii_scan_result = self.pii_scanner.scan_repo(
-                folder_path, scan_mode, files=files
-            )
-
-            custom_scan_result = self.custom_regex_scanner.scan_repo(
-                folder_path=folder_path,
-                scan_mode=scan_mode,
-                files=files,
-                custom_regex_patterns=custom_regex_patterns,
-            )
-
-        # Execute hooks
-        hooks_scan_result = self.hooks_scanner.scan_repo(
+        # Execute custom scans
+        custom_scan_results = None
+        custom_scan_results = self.custom_scanners.scan_repo(
             folder_path, scan_mode, specific_test, files=files
         )
 
+        """
+        Execute hooks only if no custom scan results were returned or if running all scans.
+        If a hook and custom scan exist with the same id, only the custom scan will run.
+        Without this check, if we specify a non-existant pre-commit hook id but a valid custom scan id,
+        the final result won't be succesful as the pre-commit command will exit with return code 1.
+        """
+        hooks_scan_results = None
+        if custom_scan_results is None or specific_test is None:
+            hooks_scan_results = self.hooks_scanner.scan_repo(
+                folder_path, scan_mode, specific_test, files=files
+            )
+
         scan_result = utilities.merge_scan_results(
-            [
-                pii_scan_result,
-                custom_scan_result,
-                hooks_scan_result,
-            ]
+            [custom_scan_results, hooks_scan_results]
         )
 
         details = scan_result.output or "Unknown output during scan"
@@ -229,13 +215,3 @@ def _get_commited_files(self, scan_mode: ScanMode) -> list[Path]:
             return [Path(file) for file in committed_files]
         except:
             return None
-
-    def _get_custom_scan_patterns(self, folder_path: Path) -> list[Path]:
-        settings = self.action_deps.settings.load(folder_path)
-        if (
-            settings.scan_patterns is not None
-            and settings.scan_patterns.custom_scan_patterns is not None
-        ):
-            custom_scan_patterns = settings.scan_patterns.custom_scan_patterns
-            return custom_scan_patterns
-        return []

secureli/container.py

@@ -1,5 +1,10 @@
 from dependency_injector import containers, providers
 
+from secureli.modules.custom_scanners.custom_regex_scanner.custom_regex_scanner import (
+    CustomRegexScannerService,
+)
+from secureli.modules.custom_scanners.custom_scans import CustomScannersService
+from secureli.modules.custom_scanners.pii_scanner.pii_scanner import PiiScannerService
 from secureli.modules.shared.abstractions.echo import TyperEcho
 from secureli.modules.shared.abstractions.lexer_guesser import PygmentsLexerGuesser
 from secureli.modules.shared.abstractions.pre_commit import PreCommitAbstraction
@@ -15,12 +20,9 @@
 from secureli.modules.shared.resources import read_resource
 from secureli.modules import language_analyzer
 from secureli.modules.observability.observability_services.logging import LoggingService
-from secureli.modules.core.core_services.scanner import HooksScannerService
+from secureli.modules.core.core_services.hook_scanner import HooksScannerService
 from secureli.modules.core.core_services.updater import UpdaterService
-from secureli.modules.pii_scanner.pii_scanner import PiiScannerService
-from secureli.modules.custom_regex_scanner.custom_regex_scanner import (
-    CustomRegexScannerService,
-)
+
 from secureli.modules.secureli_ignore import SecureliIgnoreService
 from secureli.settings import Settings
 
@@ -148,7 +150,17 @@ class Container(containers.DeclarativeContainer):
     )
 
     custom_regex_scanner_service = providers.Factory(
-        CustomRegexScannerService, repo_files=repo_files_repository, echo=echo
+        CustomRegexScannerService,
+        repo_files=repo_files_repository,
+        echo=echo,
+        settings=settings_repository,
+    )
+
+    """The service that orchestrates running custom scans (PII, Regex, etc.)"""
+    custom_scanner_service = providers.Factory(
+        CustomScannersService,
+        pii_scanner=pii_scanner_service,
+        custom_regex_scanner=custom_regex_scanner_service,
     )
 
     updater_service = providers.Factory(
@@ -190,8 +202,7 @@ class Container(containers.DeclarativeContainer):
         ScanAction,
         action_deps=action_deps,
         hooks_scanner=hooks_scanner_service,
-        pii_scanner=pii_scanner_service,
-        custom_regex_scanner=custom_regex_scanner_service,
+        custom_scanners=custom_scanner_service,
         git_repo=git_repo,
     )
 

secureli/modules/custom_regex_scanner/custom_regex_scanner.py → secureli/modules/custom_scanners/custom_regex_scanner/custom_regex_scanner.py

@@ -12,6 +12,7 @@
 import secureli.modules.shared.models.scan as scan
 from secureli.modules.shared.abstractions.echo import EchoAbstraction
 from secureli.repositories.repo_files import RepoFilesRepository
+from secureli.repositories.repo_settings import SecureliRepository
 
 
 class CustomRegexScanResult(pydantic.BaseModel):
@@ -32,15 +33,16 @@ def __init__(
         self,
         repo_files: RepoFilesRepository,
         echo: EchoAbstraction,
+        settings: SecureliRepository,
     ):
         self.repo_files = repo_files
         self.echo = echo
+        self.settings = settings
 
     def scan_repo(
         self,
         folder_path: Path,
         scan_mode: scan.ScanMode,
-        custom_regex_patterns: list[str],
         files: Optional[list[str]] = None,
     ) -> scan.ScanResult:
         """
@@ -52,6 +54,8 @@ def scan_repo(
         :return: A ScanResult object with details of whether the scan succeeded and, if not, details of the failures
         """
 
+        custom_regex_patterns = self._get_custom_scan_patterns(folder_path)
+
         file_paths = self._get_files_list(
             folder_path=folder_path, scan_mode=scan_mode, files=files
         )
@@ -206,3 +210,13 @@ def _format_string(self, str: str, formats: list[Format]) -> str:
         end = f"{RESULT_FORMAT[Format.DEFAULT]}{RESULT_FORMAT[Format.REG_WEIGHT]}"
 
         return f"{start}{str}{end}"
+
+    def _get_custom_scan_patterns(self, folder_path: Path) -> list[Path]:
+        settings = self.settings.load(folder_path)
+        if (
+            settings.scan_patterns is not None
+            and settings.scan_patterns.custom_scan_patterns is not None
+        ):
+            custom_scan_patterns = settings.scan_patterns.custom_scan_patterns
+            return custom_scan_patterns
+        return []

secureli/modules/custom_scanners/custom_scans.py

@@ -0,0 +1,73 @@
+from enum import Enum
+import logging
+from typing import Optional
+from pathlib import Path
+
+from secureli.modules.custom_scanners.custom_regex_scanner.custom_regex_scanner import (
+    CustomRegexScannerService,
+)
+from secureli.modules.custom_scanners.pii_scanner.pii_scanner import PiiScannerService
+from secureli.modules.shared import utilities
+import secureli.modules.shared.models.scan as scan
+
+
+class CustomScanId(str, Enum):
+    """
+    Scan ids of custom scans
+    """
+
+    PII = "check-pii"
+    CUSTOM_REGEX = "check-regex"
+
+
+class CustomScannersService:
+    """
+    This service orchestrates running custom scans. A custom scan is a
+    scan that is not a precommit hook scan, i.e. PII and custom regex scans.
+    """
+
+    def __init__(
+        self,
+        pii_scanner: PiiScannerService,
+        custom_regex_scanner: CustomRegexScannerService,
+    ):
+        self.pii_scanner = pii_scanner
+        self.custom_regex_scanner = custom_regex_scanner
+
+    def scan_repo(
+        self,
+        folder_path: Path,
+        scan_mode: scan.ScanMode,
+        custom_scan_id: Optional[str] = None,
+        files: Optional[str] = None,
+    ) -> scan.ScanResult:
+        # If no custom scan is specified, run all custom scans
+        if custom_scan_id is None:
+            pii_scan_result = self.pii_scanner.scan_repo(
+                folder_path, scan_mode, files=files
+            )
+            regex_scan_result = self.custom_regex_scanner.scan_repo(
+                folder_path=folder_path, scan_mode=scan_mode, files=files
+            )
+            custom_scan_results = utilities.merge_scan_results(
+                [pii_scan_result, regex_scan_result]
+            )
+            return custom_scan_results
+
+        # If the specified scan isn't known, do nothing
+        if custom_scan_id not in CustomScanId.__members__.values():
+            return None
+
+        # Run the specified custom scan only
+        scan_results = None
+        if custom_scan_id == CustomScanId.PII:
+            scan_results = self.pii_scanner.scan_repo(
+                folder_path, scan_mode, files=files
+            )
+
+        if custom_scan_id == CustomScanId.CUSTOM_REGEX:
+            scan_results = self.custom_regex_scanner.scan_repo(
+                folder_path=folder_path, scan_mode=scan_mode, files=files
+            )
+
+        return scan_results