feat: secureli modular refactor (#501)

secureli-XXX

This is a larger refactor of secureli's structure and includes the
merging of a handful of smaller tickets in an effort to silo secureli's
feature set. Additionally it includes a few enhancements, most notably
smarter language detection on merge, as well as a new PII scan feature.


## Changes
* Refactor codebase into a more modular structure
* Custom PII Scan

## Testing

* Existing tests pass, code coverage requirements are maintained, e2e
tests passing

## Clean Code Checklist
- [x] Meets acceptance criteria for issue
- [x] New logic is covered with automated tests
- [x] Appropriate exception handling added
- [x] Thoughtful logging included
- [x] Documentation is updated
- [ ] Follow-up work is documented in TODOs
- [x] TODOs have a ticket associated with them
- [x] No commented-out code included


<!--
Github-flavored markdown reference:
https://docs.github.com/en/get-started/writing-on-github
-->

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: Isaac Heist <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: github-actions <[email protected]>
Co-authored-by: Kathleen Hogan <[email protected]>
Co-authored-by: Kathleen Hogan <[email protected]>
Co-authored-by: kevin-orlando <[email protected]>
Co-authored-by: Jordan Heffernan <[email protected]>

.secureli/.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: check-yaml
     -   id: check-added-large-files
 -   repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.2.0
+    rev: 24.3.0
     hooks:
     -   id: black
 -   repo: https://github.com/yelp/detect-secrets

.vscode/launch.json

@@ -11,7 +11,7 @@
             "stopOnEntry": false,
             "console": "integratedTerminal",
             "justMyCode": true,
-            "args": ["--version"]
+            "args": ["--version", "update"]
         },
         {
             "name": "Python: secureli --help",
@@ -59,8 +59,8 @@
         },
         {
             "name": "Debug Tests",
-            "type": "python",
-            "request": "test",
+            "type": "debugpy",
+            "request": "launch",
             "program": "${file}",
             "purpose": ["debug-test"],
             "console": "integratedTerminal",

CONTRIBUTING.md

@@ -237,6 +237,10 @@ Services do not leverage 3rd Party or External Dependencies directly, not even t
 
 Unit tests of Services are done with mock services and abstractions.
 
+### Scanning Services
+
+Scanning is largely done by way of pre-commit hooks as configured in `.pre-commit-config.yaml`. However, we do also implement our own custom scans in separate Scanner Services, e.g., the PII scan. The results of these multiple scans are then merged together at the Action layer for output.
+
 ## Abstractions
 
 Abstractions are mini-services that are meant to encapsulate a 3rd-party dependency. This allows us to mock interfaces we don’t own easier than leveraging duck typing and magic mocks while testing our own logic. This also allows us to swap out the underlying dependency with less risk of disruption to our entire application.

README.md

@@ -80,6 +80,24 @@ All you need to do is run:
 
 Running `secureli init` will allow seCureLI to detect the languages in your repo, install pre-commit, install all the appropriate pre-commit hooks for your local repo, run a scan for secrets in your local repo, and update the installed hooks.
 
+## Scan
+
+To manually trigger a scan, run:
+
+```bash
+% secureli scan
+```
+
+This will run through all hooks and custom scans, unless a `--specific-test` option is used. The default is to scan staged files only. To scan all files instead, use the `--mode all-files` option.
+
+### PII Scan
+
+seCureLI utilizes its own PII scan, rather than using an existing pre-commit hook. To exclude a line from being flagged by the PII scanner, you can use a `disable-pii-scan` marker in a comment to disable the scan for that line.
+
+```
+test_var = "some dummy data I don't want scanned" # disable-pii-scan
+```
+
 # Upgrade
 
 ## Upgrading seCureLI via Homebrew

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 name = "secureli"
 version = "0.32.0"
 description = "Secure Project Manager"
-authors = ["Caleb Tonn <[email protected]>"]
+authors = ["Caleb Tonn <[email protected]>"] # disable-pii-scan
 license = "Apache-2.0"
 readme = "README.md"
 

scripts/secureli-deployment.sh

@@ -13,7 +13,7 @@ else
     echo "Pulling down the most recent published secureli release v${secureliVersion}"
     gh release download v$secureliVersion
     export secureliSha256=$(sha256sum ./secureli-${secureliVersion}.tar.gz | awk '{print $1}')
-    git config --global user.email "[email protected]"
+    git config --global user.email "[email protected]" # disable-pii-scan
     git config --global user.name "Secureli Automation"
     python ./scripts/get-secureli-dependencies.py
     cd homebrew-secureli

secureli/actions/action.py

@@ -1,49 +1,18 @@
 from abc import ABC
-from enum import Enum
 from pathlib import Path
-from typing import Optional
-from secureli.abstractions.echo import EchoAbstraction
-from secureli.consts.logging import TELEMETRY_DEFAULT_ENDPOINT
-from secureli.models.echo import Color
-from secureli.repositories.secureli_config import (
-    SecureliConfig,
-    SecureliConfigRepository,
-    VerifyConfigOutcome,
-)
-from secureli.repositories.settings import SecureliRepository, TelemetrySettings
-from secureli.services.language_analyzer import LanguageAnalyzerService, AnalyzeResult
-from secureli.services.language_config import LanguageNotSupportedError
-from secureli.services.language_support import (
-    LanguageMetadata,
-    LanguageSupportService,
-)
-from secureli.services.scanner import ScannerService, ScanMode
-from secureli.services.updater import UpdaterService
-
-import pydantic
-from secureli.utilities.formatter import format_sentence_list
-
-
-class VerifyOutcome(str, Enum):
-    INSTALL_CANCELED = "install-canceled"
-    INSTALL_FAILED = "install-failed"
-    INSTALL_SUCCEEDED = "install-succeeded"
-    UPDATE_CANCELED = "update-canceled"
-    UPDATE_SUCCEEDED = "update-succeeded"
-    UPDATE_FAILED = "update-failed"
-    UP_TO_DATE = "up-to-date"
-
-
-class VerifyResult(pydantic.BaseModel):
-    """
-    The outcomes of performing verification. Actions can use these results
-    to decide whether to proceed with their post-initialization actions or not.
-    """
+from secureli.modules.shared.abstractions.echo import EchoAbstraction
+from secureli.modules.observability.consts.logging import TELEMETRY_DEFAULT_ENDPOINT
+from secureli.modules.shared.models.echo import Color
+from secureli.modules.shared.models.install import VerifyOutcome, VerifyResult
+from secureli.modules.shared.models import language
+from secureli.modules.shared.models.scan import ScanMode
+from secureli.repositories import secureli_config
+from secureli.repositories.repo_settings import SecureliRepository, TelemetrySettings
+from secureli.modules.language_analyzer import language_analyzer, language_support
+from secureli.modules.core.core_services.scanner import HooksScannerService
+from secureli.modules.core.core_services.updater import UpdaterService
 
-    outcome: VerifyOutcome
-    config: Optional[SecureliConfig] = None
-    analyze_result: Optional[AnalyzeResult] = None
-    file_path: Optional[Path] = None
+from secureli.modules.shared.utilities import format_sentence_list
 
 
 class ActionDependencies:
@@ -56,17 +25,17 @@ class ActionDependencies:
     def __init__(
         self,
         echo: EchoAbstraction,
-        language_analyzer: LanguageAnalyzerService,
-        language_support: LanguageSupportService,
-        scanner: ScannerService,
-        secureli_config: SecureliConfigRepository,
+        language_analyzer: language_analyzer.LanguageAnalyzerService,
+        language_support: language_support.LanguageSupportService,
+        hooks_scanner: HooksScannerService,
+        secureli_config: secureli_config.SecureliConfigRepository,
         settings: SecureliRepository,
         updater: UpdaterService,
     ):
         self.echo = echo
         self.language_analyzer = language_analyzer
         self.language_support = language_support
-        self.scanner = scanner
+        self.hooks_scanner = hooks_scanner
         self.secureli_config = secureli_config
         self.settings = settings
         self.updater = updater
@@ -78,48 +47,71 @@ class Action(ABC):
     def __init__(self, action_deps: ActionDependencies):
         self.action_deps = action_deps
 
+    def get_secureli_config(self, reset: bool) -> secureli_config.SecureliConfig:
+        return (
+            secureli_config.SecureliConfig()
+            if reset
+            else self.action_deps.secureli_config.load()
+        )
+
     def verify_install(
-        self, folder_path: Path, reset: bool, always_yes: bool
+        self,
+        folder_path: Path,
+        reset: bool,
+        always_yes: bool,
+        files: list[Path],
     ) -> VerifyResult:
         """
         Installs, upgrades or verifies the current seCureLI installation
         :param folder_path: The folder path to initialize the repo for
         :param reset: If true, disregard existing configuration and start fresh
         :param always_yes: Assume "Yes" to all prompts
+        :param files: A List of files to scope the install to. This allows language
+        detection to run on only a selected list of files when scanning the repo.
         """
-
-        if self.action_deps.secureli_config.verify() == VerifyConfigOutcome.OUT_OF_DATE:
+        if (
+            self.action_deps.secureli_config.verify()
+            == secureli_config.VerifyConfigOutcome.OUT_OF_DATE
+        ):
             update_config = self._update_secureli_config_only(always_yes)
             if update_config.outcome != VerifyOutcome.UPDATE_SUCCEEDED:
                 self.action_deps.echo.error("seCureLI could not be verified.")
                 return VerifyResult(
                     outcome=update_config.outcome,
                 )
-
-        pre_commit_config_location = (
-            self.action_deps.scanner.pre_commit.get_preferred_pre_commit_config_path(
+        pre_commit_config_location_is_correct = self.action_deps.hooks_scanner.pre_commit.get_pre_commit_config_path_is_correct(
+            folder_path
+        )
+        preferred_config_path = self.action_deps.hooks_scanner.pre_commit.get_preferred_pre_commit_config_path(
+            folder_path
+        )
+        pre_commit_to_preserve = (
+            self.action_deps.hooks_scanner.pre_commit.pre_commit_config_exists(
                 folder_path
             )
+            and not pre_commit_config_location_is_correct
         )
-        if not pre_commit_config_location.exists():
+        if pre_commit_to_preserve:
             update_result: VerifyResult = (
                 self._update_secureli_pre_commit_config_location(
                     folder_path, always_yes
                 )
             )
-            pre_commit_config_location = update_result.file_path
+
             if update_result.outcome != VerifyOutcome.UPDATE_SUCCEEDED:
                 self.action_deps.echo.error(
-                    "seCureLI pre-commit-config.yaml could not be updated."
+                    "seCureLI .pre-commit-config.yaml could not be moved."
                 )
                 return update_result
+            else:
+                preferred_config_path = update_result.file_path
 
-        config = SecureliConfig() if reset else self.action_deps.secureli_config.load()
+        config = self.get_secureli_config(reset=reset)
         languages = []
 
         try:
-            languages = self._detect_languages(folder_path)
-        except (ValueError, LanguageNotSupportedError) as e:
+            languages = self._detect_languages(folder_path, files)
+        except (ValueError, language.LanguageNotSupportedError) as e:
             if config.languages and config.version_installed:
                 self.action_deps.echo.warning(
                     f"Newly detected languages are unsupported by seCureLI"
@@ -148,7 +140,7 @@ def verify_install(
                 languages,
                 newly_detected_languages,
                 always_yes,
-                pre_commit_config_location,
+                preferred_config_path if pre_commit_to_preserve else None,
             )
         else:
             self.action_deps.echo.print(
@@ -181,7 +173,6 @@ def _install_secureli(
 
         # pre-install
         new_install = len(detected_languages) == len(install_languages)
-
         should_install = self._prompt_to_install(
             install_languages, always_yes, new_install
         )
@@ -215,7 +206,7 @@ def _install_secureli(
         for error_msg in metadata.linter_config_write_errors:
             self.action_deps.echo.warning(error_msg)
 
-        config = SecureliConfig(
+        config = secureli_config.SecureliConfig(
             languages=detected_languages,
             version_installed=metadata.version,
         )
@@ -276,8 +267,8 @@ def _prompt_get_telemetry_api_url(self, always_yes: bool) -> str:
     def _run_post_install_scan(
         self,
         folder_path: Path,
-        config: SecureliConfig,
-        metadata: LanguageMetadata,
+        config: secureli_config.SecureliConfig,
+        metadata: language.LanguageMetadata,
         new_install: bool,
     ):
         """
@@ -309,7 +300,7 @@ def _run_post_install_scan(
             )
             self.action_deps.echo.print(f"running {secret_test_id}.")
 
-            scan_result = self.action_deps.scanner.scan_repo(
+            scan_result = self.action_deps.hooks_scanner.scan_repo(
                 folder_path,
                 ScanMode.ALL_FILES,
                 specific_test=secret_test_id,
@@ -322,14 +313,14 @@ def _run_post_install_scan(
                 f"{format_sentence_list(config.languages)} does not support secrets detection, skipping"
             )
 
-    def _detect_languages(self, folder_path: Path) -> list[str]:
+    def _detect_languages(self, folder_path: Path, files: list[Path]) -> list[str]:
         """
         Detects programming languages present in the repository
         :param folder_path: The folder path to initialize the repo for
         :return: A list of all languages found in the repository
         """
 
-        analyze_result = self.action_deps.language_analyzer.analyze(folder_path)
+        analyze_result = self.action_deps.language_analyzer.analyze(folder_path, files)
 
         if analyze_result.skipped_files:
             self.action_deps.echo.warning(
@@ -438,16 +429,18 @@ def _update_secureli_pre_commit_config_location(
         to avoid breaking backward compatibility.
         """
         self.action_deps.echo.print(
-            "seCureLI's .pre-commit-config.yaml is in a deprecated location."
+            "The .pre-commit-config.yaml is in a deprecated location."
         )
         response = always_yes or self.action_deps.echo.confirm(
             "Would you like it automatically moved to the .secureli/ directory?",
             default_response=True,
         )
         if response:
             try:
-                new_file_path = self.action_deps.scanner.pre_commit.migrate_config_file(
-                    folder_path
+                new_file_path = (
+                    self.action_deps.hooks_scanner.pre_commit.migrate_config_file(
+                        folder_path
+                    )
                 )
                 return VerifyResult(
                     outcome=VerifyOutcome.UPDATE_SUCCEEDED, file_path=new_file_path
@@ -456,8 +449,10 @@ def _update_secureli_pre_commit_config_location(
                 return VerifyResult(outcome=VerifyOutcome.UPDATE_FAILED)
         else:
             self.action_deps.echo.warning(".pre-commit-config.yaml migration declined")
-            deprecated_location = self.action_deps.scanner.get_pre_commit_config_path(
-                folder_path
+            deprecated_location = (
+                self.action_deps.hooks_scanner.pre_commit.get_pre_commit_config_path(
+                    folder_path
+                )
             )
             return VerifyResult(
                 outcome=VerifyOutcome.UPDATE_CANCELED, file_path=deprecated_location

secureli/actions/build.py

@@ -1,5 +1,6 @@
-from secureli.abstractions.echo import EchoAbstraction, Color
-from secureli.services.logging import LoggingService, LogAction
+from secureli.modules.shared.abstractions.echo import EchoAbstraction, Color
+from secureli.modules.observability.observability_services.logging import LoggingService
+from secureli.modules.shared.models.logging import LogAction
 
 
 class BuildAction:

secureli/actions/initializer.py

@@ -1,8 +1,10 @@
 from pathlib import Path
 
 from secureli.actions.scan import ScanAction
-from secureli.actions.action import Action, ActionDependencies, VerifyResult
-from secureli.services.logging import LoggingService, LogAction
+from secureli.actions.action import Action, ActionDependencies
+from secureli.modules.observability.observability_services.logging import LoggingService
+from secureli.modules.shared.models.install import VerifyResult
+from secureli.modules.shared.models.logging import LogAction
 
 
 class InitializerAction(Action):
@@ -25,7 +27,7 @@ def initialize_repo(
         :param reset: If true, disregard existing configuration and start fresh
         :param always_yes: Assume "Yes" to all prompts
         """
-        verify_result = self.verify_install(folder_path, reset, always_yes)
+        verify_result = self.verify_install(folder_path, reset, always_yes, files=None)
         if verify_result.outcome in ScanAction.halting_outcomes:
             self.logging.failure(LogAction.init, verify_result.outcome)
         else: