From f1e90364acee0234cb2b115e14d7a6b0c7252da3 Mon Sep 17 00:00:00 2001 From: Jordan Heffernan <44213782+JordoHeffernan@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:50:34 -0600 Subject: [PATCH] feat: secureli modular refactor (#501) secureli-XXX This is a larger refactor of secureli's structure and includes the merging of a handful of smaller tickets in an effort to silo secureli's feature set. Additionally it includes a few enhancements, most notably smarter language detection on merge, as well as a new PII scan feature. ## Changes * Refactor codebase into a more modular structure * Custom PII Scan ## Testing * Existing tests pass, code coverage requirements are maintained, e2e tests passing ## Clean Code Checklist - [x] Meets acceptance criteria for issue - [x] New logic is covered with automated tests - [x] Appropriate exception handling added - [x] Thoughtful logging included - [x] Documentation is updated - [ ] Follow-up work is documented in TODOs - [x] TODOs have a ticket associated with them - [x] No commented-out code included --------- Signed-off-by: dependabot[bot] Co-authored-by: Isaac Heist Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions Co-authored-by: Kathleen Hogan <56691584+kathleen-hogan-slalom@users.noreply.github.com> Co-authored-by: Kathleen Hogan Co-authored-by: kevin-orlando <58826693+kevin-orlando@users.noreply.github.com> Co-authored-by: Jordan Heffernan --- .secureli/.pre-commit-config.yaml | 2 +- .vscode/launch.json | 6 +- CONTRIBUTING.md | 4 + README.md | 18 ++ pyproject.toml | 2 +- scripts/secureli-deployment.sh | 2 +- secureli/actions/action.py | 141 ++++----- secureli/actions/build.py | 5 +- secureli/actions/initializer.py | 8 +- secureli/actions/scan.py | 97 ++++-- secureli/actions/setup.py | 2 +- secureli/actions/update.py | 7 +- secureli/container.py | 58 ++-- secureli/main.py | 12 +- .../{abstractions => modules}/__init__.py | 0 .../core/core_services}/scanner.py | 61 +--- .../core/core_services}/updater.py | 2 +- .../language_analyzer/__init__ .py} | 0 .../language_analyzer}/git_ignore.py | 0 .../language_analyzer}/language_analyzer.py | 29 +- .../language_analyzer}/language_config.py | 84 ++--- .../language_analyzer}/language_support.py | 167 +++------- .../observability}/consts/logging.py | 0 .../observability_services}/logging.py | 27 +- secureli/modules/pii_scanner/pii_scanner.py | 214 +++++++++++++ .../{services => modules}/secureli_ignore.py | 0 .../shared/abstractions}/__init__.py | 0 .../{ => modules/shared}/abstractions/echo.py | 21 +- .../shared}/abstractions/lexer_guesser.py | 0 .../shared}/abstractions/pre_commit.py | 19 +- secureli/modules/shared/abstractions/repo.py | 21 ++ .../modules/shared/consts}/__init__.py | 0 secureli/modules/shared/consts/language.py | 11 + secureli/modules/shared/consts/pii.py | 51 +++ .../modules/shared/models}/__init__.py | 0 secureli/modules/shared/models/config.py | 30 ++ secureli/{ => modules/shared}/models/echo.py | 14 + .../{ => modules/shared}/models/exit_codes.py | 0 secureli/modules/shared/models/install.py | 30 ++ secureli/modules/shared/models/language.py | 72 +++++ secureli/modules/shared/models/logging.py | 11 + .../shared}/models/publish_results.py | 2 +- .../{ => modules/shared}/models/result.py | 0 secureli/modules/shared/models/scan.py | 41 +++ secureli/modules/shared/resources/__init__.py | 1 + .../shared}/resources/files/build.txt | 0 .../files/configs/javascript.config.yaml | 0 .../files/configs/typescript.config.yaml | 0 .../shared}/resources/files/epilog.md | 0 .../pre-commit/base/base-pre-commit.yaml | 0 .../pre-commit/base/csharp-pre-commit.yaml | 0 .../files/pre-commit/base/go-pre-commit.yaml | 0 .../pre-commit/base/java-pre-commit.yaml | 0 .../base/javascript-pre-commit.yaml | 0 .../pre-commit/base/kotlin-pre-commit.yaml | 0 .../pre-commit/base/python-pre-commit.yaml | 0 .../pre-commit/base/swift-pre-commit.yaml | 0 .../pre-commit/base/terraform-pre-commit.yaml | 0 .../base/typescript-pre-commit.yaml | 0 .../pre-commit/lint/base-pre-commit.yaml | 0 .../pre-commit/lint/csharp-pre-commit.yaml | 0 .../files/pre-commit/lint/go-pre-commit.yaml | 0 .../pre-commit/lint/java-pre-commit.yaml | 0 .../lint/javascript-pre-commit.yaml | 0 .../pre-commit/lint/kotlin-pre-commit.yaml | 0 .../pre-commit/lint/python-pre-commit.yaml | 0 .../pre-commit/lint/swift-pre-commit.yaml | 0 .../pre-commit/lint/terraform-pre-commit.yaml | 0 .../lint/typescript-pre-commit.yaml | 0 .../pre-commit/secrets_detecting_repos.yaml | 0 .../shared}/resources/read_resource.py | 0 .../{ => modules/shared}/resources/slugify.py | 0 secureli/modules/shared/utilities.py | 176 +++++++++++ secureli/repositories/repo_files.py | 39 ++- .../{settings.py => repo_settings.py} | 6 +- secureli/resources/__init__.py | 1 - secureli/settings.py | 17 +- secureli/utilities/formatter.py | 14 - secureli/utilities/git_meta.py | 33 -- secureli/utilities/hash.py | 11 - secureli/utilities/logging.py | 15 - secureli/utilities/patterns.py | 34 -- secureli/utilities/secureli_meta.py | 6 - secureli/utilities/usage_stats.py | 58 ---- tests/actions/conftest.py | 3 +- tests/actions/test_action.py | 208 +++++++------ tests/actions/test_build_action.py | 2 +- tests/actions/test_initializer_action.py | 14 +- tests/actions/test_scan_action.py | 292 ++++++++++++++---- tests/actions/test_update_action.py | 2 +- tests/application/test_main.py | 8 +- tests/application/test_settings.py | 9 +- tests/conftest.py | 2 +- tests/end-to-end/testinstallnewhooks.bats | 22 ++ tests/end-to-end/testpreservehooks.bats | 3 +- .../.pre-commit-config.yaml | 0 tests/{services => modules}/__init__.py | 0 .../core}/test_scanner_service.py | 44 +-- .../core}/test_updater_service.py | 4 +- .../language_analyzer}/test_git_ignore.py | 44 +-- .../test_language_analyzer.py | 30 +- .../test_language_config.py | 44 ++- .../test_language_support.py | 122 ++++---- .../observability}/test_logging_service.py | 19 +- .../pii_scanner/test_pii_scanner_service.py | 129 ++++++++ .../shared/abstractions}/__init__.py | 0 .../shared}/abstractions/test_pre_commit.py | 76 +++-- .../test_pygments_lexer_guesser.py | 2 +- .../modules/shared/abstractions/test_repo.py | 25 ++ .../shared}/abstractions/test_typer_echo.py | 38 ++- tests/modules/shared/resources/__init__.py | 0 .../shared}/resources/test_read_resource.py | 4 +- tests/modules/shared/utilities/__init__.py | 0 .../shared}/utilities/test_formatter.py | 2 +- .../shared}/utilities/test_git_meta.py | 28 +- .../shared}/utilities/test_logging.py | 6 +- .../shared}/utilities/test_patterns.py | 2 +- .../shared}/utilities/test_secureli_meta.py | 4 +- .../shared}/utilities/test_usage_stats.py | 60 ++-- .../test_secureli_ignore.py | 28 +- .../test_repo_files_repository.py | 24 ++ tests/repositories/test_secureli_config.py | 60 ++-- .../repositories/test_settings_repository.py | 32 +- 123 files changed, 1967 insertions(+), 1107 deletions(-) rename secureli/{abstractions => modules}/__init__.py (100%) rename secureli/{services => modules/core/core_services}/scanner.py (82%) rename secureli/{services => modules/core/core_services}/updater.py (97%) rename secureli/{services/__init__.py => modules/language_analyzer/__init__ .py} (100%) rename secureli/{services => modules/language_analyzer}/git_ignore.py (100%) rename secureli/{services => modules/language_analyzer}/language_analyzer.py (83%) rename secureli/{services => modules/language_analyzer}/language_config.py (66%) rename secureli/{services => modules/language_analyzer}/language_support.py (72%) rename secureli/{ => modules/observability}/consts/logging.py (100%) rename secureli/{services => modules/observability/observability_services}/logging.py (83%) create mode 100644 secureli/modules/pii_scanner/pii_scanner.py rename secureli/{services => modules}/secureli_ignore.py (100%) rename secureli/{utilities => modules/shared/abstractions}/__init__.py (100%) rename secureli/{ => modules/shared}/abstractions/echo.py (90%) rename secureli/{ => modules/shared}/abstractions/lexer_guesser.py (100%) rename secureli/{ => modules/shared}/abstractions/pre_commit.py (95%) create mode 100644 secureli/modules/shared/abstractions/repo.py rename {tests/abstractions => secureli/modules/shared/consts}/__init__.py (100%) create mode 100644 secureli/modules/shared/consts/language.py create mode 100644 secureli/modules/shared/consts/pii.py rename {tests/resources => secureli/modules/shared/models}/__init__.py (100%) create mode 100644 secureli/modules/shared/models/config.py rename secureli/{ => modules/shared}/models/echo.py (60%) rename secureli/{ => modules/shared}/models/exit_codes.py (100%) create mode 100644 secureli/modules/shared/models/install.py create mode 100644 secureli/modules/shared/models/language.py create mode 100644 secureli/modules/shared/models/logging.py rename secureli/{ => modules/shared}/models/publish_results.py (80%) rename secureli/{ => modules/shared}/models/result.py (100%) create mode 100644 secureli/modules/shared/models/scan.py create mode 100644 secureli/modules/shared/resources/__init__.py rename secureli/{ => modules/shared}/resources/files/build.txt (100%) rename secureli/{ => modules/shared}/resources/files/configs/javascript.config.yaml (100%) rename secureli/{ => modules/shared}/resources/files/configs/typescript.config.yaml (100%) rename secureli/{ => modules/shared}/resources/files/epilog.md (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/base/base-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/base/csharp-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/base/go-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/base/java-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/base/javascript-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/base/kotlin-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/base/python-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/base/swift-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/base/terraform-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/base/typescript-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/lint/base-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/lint/csharp-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/lint/go-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/lint/java-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/lint/javascript-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/lint/kotlin-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/lint/python-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/lint/swift-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/lint/terraform-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/lint/typescript-pre-commit.yaml (100%) rename secureli/{ => modules/shared}/resources/files/pre-commit/secrets_detecting_repos.yaml (100%) rename secureli/{ => modules/shared}/resources/read_resource.py (100%) rename secureli/{ => modules/shared}/resources/slugify.py (100%) create mode 100644 secureli/modules/shared/utilities.py rename secureli/repositories/{settings.py => repo_settings.py} (96%) delete mode 100644 secureli/resources/__init__.py delete mode 100644 secureli/utilities/formatter.py delete mode 100644 secureli/utilities/git_meta.py delete mode 100644 secureli/utilities/hash.py delete mode 100644 secureli/utilities/logging.py delete mode 100644 secureli/utilities/patterns.py delete mode 100644 secureli/utilities/secureli_meta.py delete mode 100644 secureli/utilities/usage_stats.py create mode 100644 tests/end-to-end/testinstallnewhooks.bats rename tests/{services => modules}/.pre-commit-config.yaml (100%) rename tests/{services => modules}/__init__.py (100%) rename tests/{services => modules/core}/test_scanner_service.py (85%) rename tests/{services => modules/core}/test_updater_service.py (95%) rename tests/{services => modules/language_analyzer}/test_git_ignore.py (75%) rename tests/{services => modules/language_analyzer}/test_language_analyzer.py (77%) rename tests/{services => modules/language_analyzer}/test_language_config.py (79%) rename tests/{services => modules/language_analyzer}/test_language_support.py (81%) rename tests/{services => modules/observability}/test_logging_service.py (82%) create mode 100644 tests/modules/pii_scanner/test_pii_scanner_service.py rename tests/{utilities => modules/shared/abstractions}/__init__.py (100%) rename tests/{ => modules/shared}/abstractions/test_pre_commit.py (89%) rename tests/{ => modules/shared}/abstractions/test_pygments_lexer_guesser.py (90%) create mode 100644 tests/modules/shared/abstractions/test_repo.py rename tests/{ => modules/shared}/abstractions/test_typer_echo.py (87%) create mode 100644 tests/modules/shared/resources/__init__.py rename tests/{ => modules/shared}/resources/test_read_resource.py (89%) create mode 100644 tests/modules/shared/utilities/__init__.py rename tests/{ => modules/shared}/utilities/test_formatter.py (90%) rename tests/{ => modules/shared}/utilities/test_git_meta.py (74%) rename tests/{ => modules/shared}/utilities/test_logging.py (64%) rename tests/{ => modules/shared}/utilities/test_patterns.py (92%) rename tests/{ => modules/shared}/utilities/test_secureli_meta.py (84%) rename tests/{ => modules/shared}/utilities/test_usage_stats.py (62%) rename tests/{services => modules}/test_secureli_ignore.py (62%) diff --git a/.secureli/.pre-commit-config.yaml b/.secureli/.pre-commit-config.yaml index d98861a0..b369b74d 100644 --- a/.secureli/.pre-commit-config.yaml +++ b/.secureli/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.2.0 + rev: 24.3.0 hooks: - id: black - repo: https://github.com/yelp/detect-secrets diff --git a/.vscode/launch.json b/.vscode/launch.json index 8b160b9c..42e144b6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,7 @@ "stopOnEntry": false, "console": "integratedTerminal", "justMyCode": true, - "args": ["--version"] + "args": ["--version", "update"] }, { "name": "Python: secureli --help", @@ -59,8 +59,8 @@ }, { "name": "Debug Tests", - "type": "python", - "request": "test", + "type": "debugpy", + "request": "launch", "program": "${file}", "purpose": ["debug-test"], "console": "integratedTerminal", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ca8fa0f6..0de4319c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -237,6 +237,10 @@ Services do not leverage 3rd Party or External Dependencies directly, not even t Unit tests of Services are done with mock services and abstractions. +### Scanning Services + +Scanning is largely done by way of pre-commit hooks as configured in `.pre-commit-config.yaml`. However, we do also implement our own custom scans in separate Scanner Services, e.g., the PII scan. The results of these multiple scans are then merged together at the Action layer for output. + ## Abstractions Abstractions are mini-services that are meant to encapsulate a 3rd-party dependency. This allows us to mock interfaces we don’t own easier than leveraging duck typing and magic mocks while testing our own logic. This also allows us to swap out the underlying dependency with less risk of disruption to our entire application. diff --git a/README.md b/README.md index 1a013c74..8f4e366f 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,24 @@ All you need to do is run: Running `secureli init` will allow seCureLI to detect the languages in your repo, install pre-commit, install all the appropriate pre-commit hooks for your local repo, run a scan for secrets in your local repo, and update the installed hooks. +## Scan + +To manually trigger a scan, run: + +```bash +% secureli scan +``` + +This will run through all hooks and custom scans, unless a `--specific-test` option is used. The default is to scan staged files only. To scan all files instead, use the `--mode all-files` option. + +### PII Scan + +seCureLI utilizes its own PII scan, rather than using an existing pre-commit hook. To exclude a line from being flagged by the PII scanner, you can use a `disable-pii-scan` marker in a comment to disable the scan for that line. + +``` +test_var = "some dummy data I don't want scanned" # disable-pii-scan +``` + # Upgrade ## Upgrading seCureLI via Homebrew diff --git a/pyproject.toml b/pyproject.toml index aea87090..4a804630 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api" name = "secureli" version = "0.32.0" description = "Secure Project Manager" -authors = ["Caleb Tonn "] +authors = ["Caleb Tonn "] # disable-pii-scan license = "Apache-2.0" readme = "README.md" diff --git a/scripts/secureli-deployment.sh b/scripts/secureli-deployment.sh index 9af7969c..1fb1b6a8 100755 --- a/scripts/secureli-deployment.sh +++ b/scripts/secureli-deployment.sh @@ -13,7 +13,7 @@ else echo "Pulling down the most recent published secureli release v${secureliVersion}" gh release download v$secureliVersion export secureliSha256=$(sha256sum ./secureli-${secureliVersion}.tar.gz | awk '{print $1}') - git config --global user.email "secureli-automation@slalom.com" + git config --global user.email "secureli-automation@slalom.com" # disable-pii-scan git config --global user.name "Secureli Automation" python ./scripts/get-secureli-dependencies.py cd homebrew-secureli diff --git a/secureli/actions/action.py b/secureli/actions/action.py index 157b478b..c89b8d3e 100644 --- a/secureli/actions/action.py +++ b/secureli/actions/action.py @@ -1,49 +1,18 @@ from abc import ABC -from enum import Enum from pathlib import Path -from typing import Optional -from secureli.abstractions.echo import EchoAbstraction -from secureli.consts.logging import TELEMETRY_DEFAULT_ENDPOINT -from secureli.models.echo import Color -from secureli.repositories.secureli_config import ( - SecureliConfig, - SecureliConfigRepository, - VerifyConfigOutcome, -) -from secureli.repositories.settings import SecureliRepository, TelemetrySettings -from secureli.services.language_analyzer import LanguageAnalyzerService, AnalyzeResult -from secureli.services.language_config import LanguageNotSupportedError -from secureli.services.language_support import ( - LanguageMetadata, - LanguageSupportService, -) -from secureli.services.scanner import ScannerService, ScanMode -from secureli.services.updater import UpdaterService - -import pydantic -from secureli.utilities.formatter import format_sentence_list - - -class VerifyOutcome(str, Enum): - INSTALL_CANCELED = "install-canceled" - INSTALL_FAILED = "install-failed" - INSTALL_SUCCEEDED = "install-succeeded" - UPDATE_CANCELED = "update-canceled" - UPDATE_SUCCEEDED = "update-succeeded" - UPDATE_FAILED = "update-failed" - UP_TO_DATE = "up-to-date" - - -class VerifyResult(pydantic.BaseModel): - """ - The outcomes of performing verification. Actions can use these results - to decide whether to proceed with their post-initialization actions or not. - """ +from secureli.modules.shared.abstractions.echo import EchoAbstraction +from secureli.modules.observability.consts.logging import TELEMETRY_DEFAULT_ENDPOINT +from secureli.modules.shared.models.echo import Color +from secureli.modules.shared.models.install import VerifyOutcome, VerifyResult +from secureli.modules.shared.models import language +from secureli.modules.shared.models.scan import ScanMode +from secureli.repositories import secureli_config +from secureli.repositories.repo_settings import SecureliRepository, TelemetrySettings +from secureli.modules.language_analyzer import language_analyzer, language_support +from secureli.modules.core.core_services.scanner import HooksScannerService +from secureli.modules.core.core_services.updater import UpdaterService - outcome: VerifyOutcome - config: Optional[SecureliConfig] = None - analyze_result: Optional[AnalyzeResult] = None - file_path: Optional[Path] = None +from secureli.modules.shared.utilities import format_sentence_list class ActionDependencies: @@ -56,17 +25,17 @@ class ActionDependencies: def __init__( self, echo: EchoAbstraction, - language_analyzer: LanguageAnalyzerService, - language_support: LanguageSupportService, - scanner: ScannerService, - secureli_config: SecureliConfigRepository, + language_analyzer: language_analyzer.LanguageAnalyzerService, + language_support: language_support.LanguageSupportService, + hooks_scanner: HooksScannerService, + secureli_config: secureli_config.SecureliConfigRepository, settings: SecureliRepository, updater: UpdaterService, ): self.echo = echo self.language_analyzer = language_analyzer self.language_support = language_support - self.scanner = scanner + self.hooks_scanner = hooks_scanner self.secureli_config = secureli_config self.settings = settings self.updater = updater @@ -78,48 +47,71 @@ class Action(ABC): def __init__(self, action_deps: ActionDependencies): self.action_deps = action_deps + def get_secureli_config(self, reset: bool) -> secureli_config.SecureliConfig: + return ( + secureli_config.SecureliConfig() + if reset + else self.action_deps.secureli_config.load() + ) + def verify_install( - self, folder_path: Path, reset: bool, always_yes: bool + self, + folder_path: Path, + reset: bool, + always_yes: bool, + files: list[Path], ) -> VerifyResult: """ Installs, upgrades or verifies the current seCureLI installation :param folder_path: The folder path to initialize the repo for :param reset: If true, disregard existing configuration and start fresh :param always_yes: Assume "Yes" to all prompts + :param files: A List of files to scope the install to. This allows language + detection to run on only a selected list of files when scanning the repo. """ - - if self.action_deps.secureli_config.verify() == VerifyConfigOutcome.OUT_OF_DATE: + if ( + self.action_deps.secureli_config.verify() + == secureli_config.VerifyConfigOutcome.OUT_OF_DATE + ): update_config = self._update_secureli_config_only(always_yes) if update_config.outcome != VerifyOutcome.UPDATE_SUCCEEDED: self.action_deps.echo.error("seCureLI could not be verified.") return VerifyResult( outcome=update_config.outcome, ) - - pre_commit_config_location = ( - self.action_deps.scanner.pre_commit.get_preferred_pre_commit_config_path( + pre_commit_config_location_is_correct = self.action_deps.hooks_scanner.pre_commit.get_pre_commit_config_path_is_correct( + folder_path + ) + preferred_config_path = self.action_deps.hooks_scanner.pre_commit.get_preferred_pre_commit_config_path( + folder_path + ) + pre_commit_to_preserve = ( + self.action_deps.hooks_scanner.pre_commit.pre_commit_config_exists( folder_path ) + and not pre_commit_config_location_is_correct ) - if not pre_commit_config_location.exists(): + if pre_commit_to_preserve: update_result: VerifyResult = ( self._update_secureli_pre_commit_config_location( folder_path, always_yes ) ) - pre_commit_config_location = update_result.file_path + if update_result.outcome != VerifyOutcome.UPDATE_SUCCEEDED: self.action_deps.echo.error( - "seCureLI pre-commit-config.yaml could not be updated." + "seCureLI .pre-commit-config.yaml could not be moved." ) return update_result + else: + preferred_config_path = update_result.file_path - config = SecureliConfig() if reset else self.action_deps.secureli_config.load() + config = self.get_secureli_config(reset=reset) languages = [] try: - languages = self._detect_languages(folder_path) - except (ValueError, LanguageNotSupportedError) as e: + languages = self._detect_languages(folder_path, files) + except (ValueError, language.LanguageNotSupportedError) as e: if config.languages and config.version_installed: self.action_deps.echo.warning( f"Newly detected languages are unsupported by seCureLI" @@ -148,7 +140,7 @@ def verify_install( languages, newly_detected_languages, always_yes, - pre_commit_config_location, + preferred_config_path if pre_commit_to_preserve else None, ) else: self.action_deps.echo.print( @@ -181,7 +173,6 @@ def _install_secureli( # pre-install new_install = len(detected_languages) == len(install_languages) - should_install = self._prompt_to_install( install_languages, always_yes, new_install ) @@ -215,7 +206,7 @@ def _install_secureli( for error_msg in metadata.linter_config_write_errors: self.action_deps.echo.warning(error_msg) - config = SecureliConfig( + config = secureli_config.SecureliConfig( languages=detected_languages, version_installed=metadata.version, ) @@ -276,8 +267,8 @@ def _prompt_get_telemetry_api_url(self, always_yes: bool) -> str: def _run_post_install_scan( self, folder_path: Path, - config: SecureliConfig, - metadata: LanguageMetadata, + config: secureli_config.SecureliConfig, + metadata: language.LanguageMetadata, new_install: bool, ): """ @@ -309,7 +300,7 @@ def _run_post_install_scan( ) self.action_deps.echo.print(f"running {secret_test_id}.") - scan_result = self.action_deps.scanner.scan_repo( + scan_result = self.action_deps.hooks_scanner.scan_repo( folder_path, ScanMode.ALL_FILES, specific_test=secret_test_id, @@ -322,14 +313,14 @@ def _run_post_install_scan( f"{format_sentence_list(config.languages)} does not support secrets detection, skipping" ) - def _detect_languages(self, folder_path: Path) -> list[str]: + def _detect_languages(self, folder_path: Path, files: list[Path]) -> list[str]: """ Detects programming languages present in the repository :param folder_path: The folder path to initialize the repo for :return: A list of all languages found in the repository """ - analyze_result = self.action_deps.language_analyzer.analyze(folder_path) + analyze_result = self.action_deps.language_analyzer.analyze(folder_path, files) if analyze_result.skipped_files: self.action_deps.echo.warning( @@ -438,7 +429,7 @@ def _update_secureli_pre_commit_config_location( to avoid breaking backward compatibility. """ self.action_deps.echo.print( - "seCureLI's .pre-commit-config.yaml is in a deprecated location." + "The .pre-commit-config.yaml is in a deprecated location." ) response = always_yes or self.action_deps.echo.confirm( "Would you like it automatically moved to the .secureli/ directory?", @@ -446,8 +437,10 @@ def _update_secureli_pre_commit_config_location( ) if response: try: - new_file_path = self.action_deps.scanner.pre_commit.migrate_config_file( - folder_path + new_file_path = ( + self.action_deps.hooks_scanner.pre_commit.migrate_config_file( + folder_path + ) ) return VerifyResult( outcome=VerifyOutcome.UPDATE_SUCCEEDED, file_path=new_file_path @@ -456,8 +449,10 @@ def _update_secureli_pre_commit_config_location( return VerifyResult(outcome=VerifyOutcome.UPDATE_FAILED) else: self.action_deps.echo.warning(".pre-commit-config.yaml migration declined") - deprecated_location = self.action_deps.scanner.get_pre_commit_config_path( - folder_path + deprecated_location = ( + self.action_deps.hooks_scanner.pre_commit.get_pre_commit_config_path( + folder_path + ) ) return VerifyResult( outcome=VerifyOutcome.UPDATE_CANCELED, file_path=deprecated_location diff --git a/secureli/actions/build.py b/secureli/actions/build.py index 7a49b961..5a6b9431 100644 --- a/secureli/actions/build.py +++ b/secureli/actions/build.py @@ -1,5 +1,6 @@ -from secureli.abstractions.echo import EchoAbstraction, Color -from secureli.services.logging import LoggingService, LogAction +from secureli.modules.shared.abstractions.echo import EchoAbstraction, Color +from secureli.modules.observability.observability_services.logging import LoggingService +from secureli.modules.shared.models.logging import LogAction class BuildAction: diff --git a/secureli/actions/initializer.py b/secureli/actions/initializer.py index 034cfb1a..a07b8bb3 100644 --- a/secureli/actions/initializer.py +++ b/secureli/actions/initializer.py @@ -1,8 +1,10 @@ from pathlib import Path from secureli.actions.scan import ScanAction -from secureli.actions.action import Action, ActionDependencies, VerifyResult -from secureli.services.logging import LoggingService, LogAction +from secureli.actions.action import Action, ActionDependencies +from secureli.modules.observability.observability_services.logging import LoggingService +from secureli.modules.shared.models.install import VerifyResult +from secureli.modules.shared.models.logging import LogAction class InitializerAction(Action): @@ -25,7 +27,7 @@ def initialize_repo( :param reset: If true, disregard existing configuration and start fresh :param always_yes: Assume "Yes" to all prompts """ - verify_result = self.verify_install(folder_path, reset, always_yes) + verify_result = self.verify_install(folder_path, reset, always_yes, files=None) if verify_result.outcome in ScanAction.halting_outcomes: self.logging.failure(LogAction.init, verify_result.outcome) else: diff --git a/secureli/actions/scan.py b/secureli/actions/scan.py index 0020912b..38613385 100644 --- a/secureli/actions/scan.py +++ b/secureli/actions/scan.py @@ -3,29 +3,27 @@ from pathlib import Path from time import time from typing import Optional - -from secureli.abstractions.echo import EchoAbstraction -from secureli.actions.action import ( - VerifyOutcome, - Action, - ActionDependencies, - VerifyResult, -) -from secureli.models.exit_codes import ExitCode -from secureli.models.publish_results import PublishResultsOption -from secureli.models.result import Result -from secureli.services.logging import LoggingService, LogAction -from secureli.services.scanner import ( - ScanMode, - ScannerService, -) +from git import Repo + +from secureli.modules.shared.abstractions.echo import EchoAbstraction +from secureli.actions import action +from secureli.modules.shared.abstractions.repo import GitRepo +from secureli.modules.shared.models.exit_codes import ExitCode +from secureli.modules.shared.models.install import VerifyOutcome, VerifyResult +from secureli.modules.shared.models.logging import LogAction +from secureli.modules.shared.models.publish_results import PublishResultsOption +from secureli.modules.shared.models.result import Result +from secureli.modules.observability.observability_services.logging import LoggingService +from secureli.modules.core.core_services.scanner import HooksScannerService +from secureli.modules.pii_scanner.pii_scanner import PiiScannerService +from secureli.modules.shared.models.scan import ScanMode, ScanResult from secureli.settings import Settings -from secureli.utilities.usage_stats import post_log, convert_failures_to_failure_count +from secureli.modules.shared import utilities ONE_WEEK_IN_SECONDS: int = 7 * 24 * 60 * 60 -class ScanAction(Action): +class ScanAction(action.Action): """The action for the secureli `scan` command, orchestrating services and outputs results""" """Any verification outcomes that would cause us to not proceed to scan.""" @@ -38,15 +36,19 @@ class ScanAction(Action): def __init__( self, - action_deps: ActionDependencies, + action_deps: action.ActionDependencies, echo: EchoAbstraction, logging: LoggingService, - scanner: ScannerService, + hooks_scanner: HooksScannerService, + pii_scanner: PiiScannerService, + git_repo: GitRepo, ): super().__init__(action_deps) - self.scanner = scanner + self.hooks_scanner = hooks_scanner + self.pii_scanner = pii_scanner self.echo = echo self.logging = logging + self.git_repo = git_repo def _check_secureli_hook_updates(self, folder_path: Path) -> VerifyResult: """ @@ -55,10 +57,12 @@ def _check_secureli_hook_updates(self, folder_path: Path) -> VerifyResult: :param folder_path: The folder path containing the .secureli/ folder """ - self.action_deps.echo.print("Checking for pre-commit hook updates...") - pre_commit_config = self.scanner.pre_commit.get_pre_commit_config(folder_path) + self.action_deps.echo.info("Checking for pre-commit hook updates...") + pre_commit_config = self.hooks_scanner.pre_commit.get_pre_commit_config( + folder_path + ) - repos_to_update = self.scanner.pre_commit.check_for_hook_updates( + repos_to_update = self.hooks_scanner.pre_commit.check_for_hook_updates( pre_commit_config ) @@ -92,7 +96,7 @@ def publish_results( publish_results_condition == PublishResultsOption.ON_FAIL and not action_successful ): - result = post_log(log_str, Settings()) + result = utilities.post_log(log_str, Settings()) self.echo.debug(result.result_message) if result.result == Result.SUCCESS: @@ -100,6 +104,24 @@ def publish_results( else: self.logging.failure(LogAction.publish, result.result_message) + def get_commited_files(self, scan_mode: ScanMode) -> list[Path]: + """ + Attempts to build a list of commited files for use in language detection if + the user is scanning staged files for an existing installation + :param scan_mode: Determines which files are scanned in the repo (i.e. staged only or all) + :returns: a list of Path objects for the commited files + """ + config = self.get_secureli_config(reset=False) + installed = bool(config.languages and config.version_installed) + + if not installed or scan_mode != ScanMode.STAGED_ONLY: + return None + try: + committed_files = self.git_repo.get_commit_diff() + return [Path(file) for file in committed_files] + except: + return None + def scan_repo( self, folder_path: Path, @@ -119,7 +141,16 @@ def scan_repo( :param specific_test: If set, limits scanning to the single pre-commit hook. Otherwise, scans with all hooks. """ - verify_result = self.verify_install(folder_path, False, always_yes) + + scan_files = [Path(file) for file in files or []] or self.get_commited_files( + scan_mode + ) + verify_result = self.verify_install( + folder_path, + False, + always_yes, + scan_files, + ) # Check if pre-commit hooks are up-to-date secureli_config = self.action_deps.secureli_config.load() @@ -132,10 +163,20 @@ def scan_repo( if verify_result.outcome in self.halting_outcomes: return - scan_result = self.scanner.scan_repo( + # Execute PII scan (unless `specific_test` is provided, in which case it will be for a hook below) + pii_scan_result: ScanResult | None = None + if not specific_test: + pii_scan_result = self.pii_scanner.scan_repo( + folder_path, scan_mode, files=files + ) + + # Execute hooks + hooks_scan_result = self.hooks_scanner.scan_repo( folder_path, scan_mode, specific_test, files=files ) + scan_result = utilities.merge_scan_results([pii_scan_result, hooks_scan_result]) + details = scan_result.output or "Unknown output during scan" self.echo.print(details) @@ -144,7 +185,7 @@ def scan_repo( [ob.__dict__ for ob in scan_result.failures] ) - individual_failure_count = convert_failures_to_failure_count( + individual_failure_count = utilities.convert_failures_to_failure_count( scan_result.failures ) diff --git a/secureli/actions/setup.py b/secureli/actions/setup.py index 68990595..c8150d18 100644 --- a/secureli/actions/setup.py +++ b/secureli/actions/setup.py @@ -1,6 +1,6 @@ import jinja2 -from secureli.services.language_support import supported_languages +from secureli.modules.shared.consts.language import supported_languages class SetupAction: diff --git a/secureli/actions/update.py b/secureli/actions/update.py index 8cdc350c..7498bc5b 100644 --- a/secureli/actions/update.py +++ b/secureli/actions/update.py @@ -1,11 +1,12 @@ from typing import Optional from pathlib import Path -from secureli.abstractions.echo import EchoAbstraction -from secureli.services.logging import LoggingService, LogAction -from secureli.services.updater import UpdaterService +from secureli.modules.shared.abstractions.echo import EchoAbstraction +from secureli.modules.observability.observability_services.logging import LoggingService +from secureli.modules.core.core_services.updater import UpdaterService from secureli.actions.action import Action, ActionDependencies from rich.progress import Progress +from secureli.modules.shared.models.logging import LogAction class UpdateAction(Action): diff --git a/secureli/container.py b/secureli/container.py index be7a5d8a..38d5b1da 100644 --- a/secureli/container.py +++ b/secureli/container.py @@ -1,25 +1,24 @@ from dependency_injector import containers, providers -from secureli.abstractions.echo import TyperEcho -from secureli.abstractions.lexer_guesser import PygmentsLexerGuesser -from secureli.abstractions.pre_commit import PreCommitAbstraction +from secureli.modules.shared.abstractions.echo import TyperEcho +from secureli.modules.shared.abstractions.lexer_guesser import PygmentsLexerGuesser +from secureli.modules.shared.abstractions.pre_commit import PreCommitAbstraction from secureli.actions.action import ActionDependencies from secureli.actions.initializer import InitializerAction from secureli.actions.scan import ScanAction from secureli.actions.build import BuildAction from secureli.actions.update import UpdateAction +from secureli.modules.shared.abstractions.repo import GitRepo from secureli.repositories.repo_files import RepoFilesRepository from secureli.repositories.secureli_config import SecureliConfigRepository -from secureli.repositories.settings import SecureliRepository -from secureli.resources import read_resource -from secureli.services.git_ignore import GitIgnoreService -from secureli.services.language_analyzer import LanguageAnalyzerService -from secureli.services.language_support import LanguageSupportService -from secureli.services.logging import LoggingService -from secureli.services.scanner import ScannerService -from secureli.services.updater import UpdaterService -from secureli.services.secureli_ignore import SecureliIgnoreService -from secureli.services.language_config import LanguageConfigService +from secureli.repositories.repo_settings import SecureliRepository +from secureli.modules.shared.resources import read_resource +from secureli.modules import language_analyzer +from secureli.modules.observability.observability_services.logging import LoggingService +from secureli.modules.core.core_services.scanner import HooksScannerService +from secureli.modules.core.core_services.updater import UpdaterService +from secureli.modules.pii_scanner.pii_scanner import PiiScannerService +from secureli.modules.secureli_ignore import SecureliIgnoreService from secureli.settings import Settings @@ -42,7 +41,7 @@ class Container(containers.DeclarativeContainer): )().ignored_file_patterns() git_ignored_file_patterns = providers.Factory( - GitIgnoreService + language_analyzer.git_ignore.GitIgnoreService )().ignored_file_patterns() combined_ignored_file_patterns = list( @@ -85,6 +84,9 @@ class Container(containers.DeclarativeContainer): echo=echo, ) + """Wraps the execution and management of git commands""" + git_repo = providers.Factory(GitRepo) + # Services """Analyzes a set of files to try to determine the most common languages""" @@ -93,10 +95,12 @@ class Container(containers.DeclarativeContainer): Manages the repository's git ignore file, making sure secureli-managed files are ignored """ - git_ignore_service = providers.Factory(GitIgnoreService) + git_ignore_service = providers.Factory( + language_analyzer.git_ignore.GitIgnoreService + ) language_config_service = providers.Factory( - LanguageConfigService, + language_analyzer.language_config.LanguageConfigService, data_loader=read_resource, command_timeout_seconds=config.language_support.command_timeout_seconds, ignored_file_patterns=secureli_ignored_file_patterns, @@ -104,7 +108,7 @@ class Container(containers.DeclarativeContainer): """Identifies the configuration version for the language and installs it""" language_support_service = providers.Factory( - LanguageSupportService, + language_analyzer.language_support.LanguageSupportService, pre_commit_hook=pre_commit_abstraction, git_ignore=git_ignore_service, language_config=language_config_service, @@ -114,7 +118,7 @@ class Container(containers.DeclarativeContainer): """Analyzes a given repo to try to identify the most common language""" language_analyzer_service = providers.Factory( - LanguageAnalyzerService, + language_analyzer.language_analyzer.LanguageAnalyzerService, repo_files=repo_files_repository, lexer_guesser=lexer_guesser, ) @@ -127,11 +131,18 @@ class Container(containers.DeclarativeContainer): ) """The service that scans the repository using pre-commit configuration""" - scanner_service = providers.Factory( - ScannerService, + hooks_scanner_service = providers.Factory( + HooksScannerService, pre_commit=pre_commit_abstraction, ) + """The service that scans the repository for potential PII""" + pii_scanner_service = providers.Factory( + PiiScannerService, + repo_files=repo_files_repository, + echo=echo, + ) + updater_service = providers.Factory( UpdaterService, pre_commit=pre_commit_abstraction, @@ -145,7 +156,7 @@ class Container(containers.DeclarativeContainer): echo=echo, language_analyzer=language_analyzer_service, language_support=language_support_service, - scanner=scanner_service, + hooks_scanner=hooks_scanner_service, secureli_config=secureli_config_repository, settings=settings_repository, updater=updater_service, @@ -172,8 +183,9 @@ class Container(containers.DeclarativeContainer): action_deps=action_deps, echo=echo, logging=logging_service, - scanner=scanner_service, - # settings_repository=settings_repository, + hooks_scanner=hooks_scanner_service, + pii_scanner=pii_scanner_service, + git_repo=git_repo, ) """Update Action, representing what happens when the update command is invoked""" diff --git a/secureli/main.py b/secureli/main.py index a4207cd9..6cf8ff37 100644 --- a/secureli/main.py +++ b/secureli/main.py @@ -3,17 +3,17 @@ from typing_extensions import Annotated import typer from typer import Option -from secureli.actions.action import VerifyOutcome -from secureli.actions.scan import ScanMode from secureli.actions.setup import SetupAction from secureli.container import Container -from secureli.models.echo import Color -from secureli.models.publish_results import PublishResultsOption -from secureli.resources import read_resource +from secureli.modules.shared.models.echo import Color +from secureli.modules.shared.models.install import VerifyOutcome +from secureli.modules.shared.models.publish_results import PublishResultsOption +from secureli.modules.shared.models.scan import ScanMode +from secureli.modules.shared.resources import read_resource from secureli.settings import Settings import secureli.repositories.secureli_config as SecureliConfig -from secureli.utilities.secureli_meta import secureli_version +from secureli.modules.shared.utilities import secureli_version # Create SetupAction outside of DI, as it's not yet available. setup_action = SetupAction(epilog_template_data=read_resource("epilog.md")) diff --git a/secureli/abstractions/__init__.py b/secureli/modules/__init__.py similarity index 100% rename from secureli/abstractions/__init__.py rename to secureli/modules/__init__.py diff --git a/secureli/services/scanner.py b/secureli/modules/core/core_services/scanner.py similarity index 82% rename from secureli/services/scanner.py rename to secureli/modules/core/core_services/scanner.py index 9ccff52d..c706207e 100644 --- a/secureli/services/scanner.py +++ b/secureli/modules/core/core_services/scanner.py @@ -2,20 +2,11 @@ from typing import Optional from pathlib import Path -import pydantic import re -from secureli.abstractions.pre_commit import PreCommitAbstraction -from secureli.repositories.settings import PreCommitSettings - - -class ScanMode(str, Enum): - """ - Which scan mode to run as when we perform scanning. - """ - - STAGED_ONLY = "staged-only" - ALL_FILES = "all-files" +import secureli.modules.shared.models.scan as scan +from secureli.modules.shared.abstractions.pre_commit import PreCommitAbstraction +from secureli.repositories.repo_settings import PreCommitSettings class OutputParseErrors(str, Enum): @@ -26,35 +17,7 @@ class OutputParseErrors(str, Enum): REPO_NOT_FOUND = "repo-not-found" -class Failure(pydantic.BaseModel): - """ - Represents the details of a failed rule from a scan - """ - - repo: str - id: str - file: str - - -class ScanResult(pydantic.BaseModel): - """ - The results of calling scan_repo - """ - - successful: bool - output: Optional[str] = None - failures: list[Failure] - - -class ScanOuput(pydantic.BaseModel): - """ - Represents the parsed output from a scan - """ - - failures: list[Failure] - - -class ScannerService: +class HooksScannerService: """ Scans the repo according to the repo's seCureLI config """ @@ -65,10 +28,10 @@ def __init__(self, pre_commit: PreCommitAbstraction): def scan_repo( self, folder_path: Path, - scan_mode: ScanMode, + scan_mode: scan.ScanMode, specific_test: Optional[str] = None, files: Optional[str] = None, - ) -> ScanResult: + ) -> scan.ScanResult: """ Scans the repo according to the repo's seCureLI config :param scan_mode: Whether to scan the staged files (i.e., the files about to be @@ -77,7 +40,7 @@ def scan_repo( If None, run all hooks. :return: A ScanResult object containing whether we succeeded and any error """ - all_files = True if scan_mode == ScanMode.ALL_FILES else False + all_files = True if scan_mode == scan.ScanMode.ALL_FILES else False execute_result = self.pre_commit.execute_hooks( folder_path, all_files, hook_id=specific_test, files=files ) @@ -85,19 +48,19 @@ def scan_repo( folder_path, output=execute_result.output ) - return ScanResult( + return scan.ScanResult( successful=execute_result.successful, output=execute_result.output, failures=parsed_output.failures, ) - def _parse_scan_ouput(self, folder_path: Path, output: str = "") -> ScanOuput: + def _parse_scan_ouput(self, folder_path: Path, output: str = "") -> scan.ScanOutput: """ Parses the output from a scan and returns a list of Failure objects representing any hook rule failures during a scan. :param folder_path: folder containing .secureli folder, usually repository root :param output: Raw output from a scan. - :return: ScanOuput object representing a list of hook rule Failure objects. + :return: ScanOutput object representing a list of hook rule Failure objects. """ failures = [] failure_indexes = [] @@ -128,9 +91,9 @@ def _parse_scan_ouput(self, folder_path: Path, output: str = "") -> ScanOuput: files = self._find_file_names(failure_output_list=failure_output_list) for file in files: - failures.append(Failure(id=id, file=file, repo=repo)) + failures.append(scan.ScanFailure(id=id, file=file, repo=repo)) - return ScanOuput(failures=failures) + return scan.ScanOutput(failures=failures) def _get_single_failure_output( self, failure_start: int, output_by_line: list[str] diff --git a/secureli/services/updater.py b/secureli/modules/core/core_services/updater.py similarity index 97% rename from secureli/services/updater.py rename to secureli/modules/core/core_services/updater.py index 4d36806c..9ebcc202 100644 --- a/secureli/services/updater.py +++ b/secureli/modules/core/core_services/updater.py @@ -2,7 +2,7 @@ from pathlib import Path import pydantic -from secureli.abstractions.pre_commit import PreCommitAbstraction +from secureli.modules.shared.abstractions.pre_commit import PreCommitAbstraction from secureli.repositories.secureli_config import SecureliConfigRepository diff --git a/secureli/services/__init__.py b/secureli/modules/language_analyzer/__init__ .py similarity index 100% rename from secureli/services/__init__.py rename to secureli/modules/language_analyzer/__init__ .py diff --git a/secureli/services/git_ignore.py b/secureli/modules/language_analyzer/git_ignore.py similarity index 100% rename from secureli/services/git_ignore.py rename to secureli/modules/language_analyzer/git_ignore.py diff --git a/secureli/services/language_analyzer.py b/secureli/modules/language_analyzer/language_analyzer.py similarity index 83% rename from secureli/services/language_analyzer.py rename to secureli/modules/language_analyzer/language_analyzer.py index f1ff540e..49c5b110 100644 --- a/secureli/services/language_analyzer.py +++ b/secureli/modules/language_analyzer/language_analyzer.py @@ -1,29 +1,10 @@ from collections import defaultdict from pathlib import Path -import pydantic - -from secureli.abstractions.lexer_guesser import LexerGuesser +from secureli.modules.shared.abstractions.lexer_guesser import LexerGuesser +from secureli.modules.shared.models.language import AnalyzeResult, SkippedFile from secureli.repositories.repo_files import RepoFilesRepository -from secureli.services.language_support import supported_languages - - -class SkippedFile(pydantic.BaseModel): - """ - A file skipped by the analysis phase. - """ - - file_path: Path - error_message: str - - -class AnalyzeResult(pydantic.BaseModel): - """ - The result of the analysis phase. - """ - - language_proportions: dict[str, float] - skipped_files: list[SkippedFile] +from secureli.modules.shared.consts.language import supported_languages class LanguageAnalyzerService: @@ -39,7 +20,7 @@ def __init__( self.repo_files = repo_files self.lexer_guesser = lexer_guesser - def analyze(self, folder_path: Path) -> AnalyzeResult: + def analyze(self, folder_path: Path, files: list[Path]) -> AnalyzeResult: """ Analyzes the folder structure and lists languages found :param folder_path: The path to the repository to analyze @@ -48,7 +29,7 @@ def analyze(self, folder_path: Path) -> AnalyzeResult: 40% of the repo is JavaScript, the result will be a dictionary containing keys "Python" and "JavaScript" with values 0.6 and 0.4 respectively """ - file_paths = self.repo_files.list_repo_files(folder_path) + file_paths = files if files else self.repo_files.list_repo_files(folder_path) results = defaultdict(int) skipped_files = [] diff --git a/secureli/services/language_config.py b/secureli/modules/language_analyzer/language_config.py similarity index 66% rename from secureli/services/language_config.py rename to secureli/modules/language_analyzer/language_config.py index 0c4b701f..4eb652c2 100644 --- a/secureli/services/language_config.py +++ b/secureli/modules/language_analyzer/language_config.py @@ -1,36 +1,10 @@ from pathlib import Path -from typing import Callable, Any - -import pydantic +from typing import Callable import yaml -from secureli.resources.slugify import slugify -from secureli.utilities.hash import hash_config -from secureli.utilities.patterns import combine_patterns - - -class LanguageNotSupportedError(Exception): - """The given language was not supported by the PreCommitHooks abstraction""" - - pass - - -class LoadLinterConfigsResult(pydantic.BaseModel): - """Results from finding and loading any pre-commit configs for the language""" - - successful: bool - linter_data: list[Any] - - -class LanguagePreCommitResult(pydantic.BaseModel): - """ - A configuration model for a supported pre-commit-configurable language. - """ - - language: str - config_data: str - version: str - linter_config: LoadLinterConfigsResult +from secureli.modules.shared.models import language +from secureli.modules.shared.resources.slugify import slugify +from secureli.modules.shared.utilities import combine_patterns, hash_config class LanguageConfigService: @@ -45,8 +19,8 @@ def __init__( self.ignored_file_patterns = ignored_file_patterns def get_language_config( - self, language: str, include_linter: bool - ) -> LanguagePreCommitResult: + self, specified_language: str, include_linter: bool + ) -> language.LanguagePreCommitResult: """ Calculates a hash of the pre-commit file for the given language to be used as part of the overall installed configuration. @@ -58,27 +32,29 @@ def get_language_config( """ try: config_data = self._calculate_combined_configuration_data( - language, include_linter + specified_language, include_linter ) linter_config_data = ( - self._load_linter_config_file(language) + self._load_linter_config_file(specified_language) if include_linter - else LoadLinterConfigsResult(successful=True, linter_data=list()) + else language.LoadLinterConfigsResult( + successful=True, linter_data=list() + ) ) version = hash_config(config_data) - return LanguagePreCommitResult( - language=language, + return language.LanguagePreCommitResult( + language=specified_language, config_data=config_data, version=version, linter_config=linter_config_data, ) except ValueError: - raise LanguageNotSupportedError( - f"Language '{language}' is currently unsupported" + raise language.LanguageNotSupportedError( + f"Language '{specified_language}' is currently unsupported" ) def _calculate_combined_configuration( - self, language: str, include_linter: bool + self, specified_language: str, include_linter: bool ) -> dict: """ Combine elements of our configuration for the specified language along with @@ -90,7 +66,7 @@ def _calculate_combined_configuration( :return: The combined configuration data as a dictionary """ config = {"repos": []} - slugified_language = slugify(language) + slugified_language = slugify(specified_language) config_folder_names = ["base"] if include_linter: @@ -110,7 +86,7 @@ def _calculate_combined_configuration( return config def _calculate_combined_configuration_data( - self, language: str, include_linter: bool + self, specified_language: str, include_linter: bool ) -> str: """ Combine elements of our configuration for the specified language along with @@ -120,10 +96,14 @@ def _calculate_combined_configuration_data( :param include_linter: Whether or not linter pre-commit hooks should be included :return: The combined configuration data as a string """ - config = self._calculate_combined_configuration(language, include_linter) + config = self._calculate_combined_configuration( + specified_language, include_linter + ) return yaml.dump(config) - def _load_linter_config_file(self, language: str) -> LoadLinterConfigsResult: + def _load_linter_config_file( + self, specified_language: str + ) -> language.LoadLinterConfigsResult: """ Load any config files for given language if they exist. :param language: repo language @@ -131,14 +111,16 @@ def _load_linter_config_file(self, language: str) -> LoadLinterConfigsResult: """ # respective name for config file - language_config_name = Path(f"configs/{slugify(language)}.config.yaml") + language_config_name = Path( + f"configs/{slugify(specified_language)}.config.yaml" + ) # build absolute path to config file if one exists - absolute_secureli_path = f'{Path(f"{__file__}").parent.resolve()}'.rsplit( - "/", 1 - )[0] + absolute_secureli_path = ( + f'{Path(f"{__file__}").parent.parent.resolve()}'.rsplit("/", 1)[0] + ) absolute_configs_path = Path( - f"{absolute_secureli_path}/resources/files/{language_config_name}" + f"{absolute_secureli_path}/modules/shared/resources/files/{language_config_name}" ) # check if config file exists for current language @@ -146,8 +128,8 @@ def _load_linter_config_file(self, language: str) -> LoadLinterConfigsResult: language_configs_data = self.data_loader(language_config_name) language_configs = yaml.safe_load_all(language_configs_data) - return LoadLinterConfigsResult( + return language.LoadLinterConfigsResult( successful=True, linter_data=language_configs ) - return LoadLinterConfigsResult(successful=False, linter_data=list()) + return language.LoadLinterConfigsResult(successful=False, linter_data=list()) diff --git a/secureli/services/language_support.py b/secureli/modules/language_analyzer/language_support.py similarity index 72% rename from secureli/services/language_support.py rename to secureli/modules/language_analyzer/language_support.py index a37909dd..c6ce8d9d 100644 --- a/secureli/services/language_support.py +++ b/secureli/modules/language_analyzer/language_support.py @@ -1,98 +1,15 @@ from pathlib import Path -from typing import Callable, Iterable, Optional, Any +from typing import Callable, Iterable, Optional -import pydantic import yaml -from secureli.abstractions.echo import EchoAbstraction +from secureli.modules.shared.models.config import HookConfiguration, LinterConfig, Repo +from secureli.modules.shared.models import language import secureli.repositories.secureli_config as SecureliConfig -from secureli.abstractions.pre_commit import PreCommitAbstraction -from secureli.services.git_ignore import GitIgnoreService -from secureli.services.language_config import LanguageConfigService -from secureli.utilities.hash import hash_config - -supported_languages = [ - "C#", - "Python", - "Java", - "Terraform", - "TypeScript", - "JavaScript", - "Go", - "Swift", - "Kotlin", -] - - -class LanguageMetadata(pydantic.BaseModel): - version: str - security_hook_id: Optional[str] - linter_config_write_errors: Optional[list[str]] = [] - - -class ValidateConfigResult(pydantic.BaseModel): - """ - The results of calling validate_config - """ - - successful: bool - output: str - - -class Repo(pydantic.BaseModel): - """A repository containing pre-commit hooks""" - - repo: str - revision: str - hooks: list[str] - - -class HookConfiguration(pydantic.BaseModel): - """A simplified pre-commit configuration representation for logging purposes""" - - repos: list[Repo] - - -class UnexpectedReposResult(pydantic.BaseModel): - """ - The result of checking for unexpected repos in config - """ - - missing_repos: Optional[list[str]] = [] - unexpected_repos: Optional[list[str]] = [] - - -class LinterConfigData(pydantic.BaseModel): - """ - Represents the structure of a linter config file - """ - - filename: str - settings: Any - - -class LinterConfig(pydantic.BaseModel): - language: str - linter_data: list[LinterConfigData] - - -class BuildConfigResult(pydantic.BaseModel): - """Result about building config for all laguages""" - - successful: bool - languages_added: list[str] - config_data: dict - linter_configs: list[LinterConfig] - version: str - - -class LinterConfigWriteResult(pydantic.BaseModel): - """ - Result from writing linter config files - """ - - successful_languages: list[str] - error_messages: list[str] +from secureli.modules.shared.abstractions.pre_commit import PreCommitAbstraction +from secureli.modules.shared.abstractions.echo import EchoAbstraction +from secureli.modules.language_analyzer import git_ignore, language_config +from secureli.modules.shared.utilities import hash_config class LanguageSupportService: @@ -104,8 +21,8 @@ class LanguageSupportService: def __init__( self, pre_commit_hook: PreCommitAbstraction, - language_config: LanguageConfigService, - git_ignore: GitIgnoreService, + language_config: language_config.LanguageConfigService, + git_ignore: git_ignore.GitIgnoreService, data_loader: Callable[[str], str], echo: EchoAbstraction, ): @@ -118,9 +35,9 @@ def __init__( def apply_support( self, languages: list[str], - language_config_result: BuildConfigResult, + language_config_result: language.BuildConfigResult, overwrite_pre_commit: bool, - ) -> LanguageMetadata: + ) -> language.LanguageMetadata: """ Applies Secure Build support for the provided languages :param languages: list of languages to provide support for @@ -131,8 +48,10 @@ def apply_support( as well as a secret-detection hook ID, if present. """ - path_to_pre_commit_file: Path = self.pre_commit_hook.get_pre_commit_config_path( - SecureliConfig.FOLDER_PATH + path_to_pre_commit_file: Path = ( + self.pre_commit_hook.get_preferred_pre_commit_config_path( + SecureliConfig.FOLDER_PATH + ) ) linter_config_write_result = self._write_pre_commit_configs( @@ -151,7 +70,7 @@ def apply_support( # Add .secureli/ to the gitignore folder if needed self.git_ignore.ignore_secureli_files() - return LanguageMetadata( + return language.LanguageMetadata( version=language_config_result.version, security_hook_id=self.secret_detection_hook_id(languages), linter_config_write_errors=linter_config_write_result.error_messages, @@ -211,13 +130,7 @@ def get_configuration(self, languages: list[str]) -> HookConfiguration: """ config = self.build_pre_commit_config(languages, set(languages)).config_data - create_repo: Callable[[Repo], Repo] = lambda raw_repo: Repo( - repo=raw_repo.get("repo", "unknown"), - revision=raw_repo.get("rev", "unknown"), - hooks=[hook.get("id", "unknown") for hook in raw_repo.get("hooks", [])], - ) - - repos = [create_repo(raw_repo) for raw_repo in config.get("repos", [])] + repos = [self._create_repo(raw_repo) for raw_repo in config.get("repos", [])] return HookConfiguration(repos=repos) def build_pre_commit_config( @@ -225,13 +138,13 @@ def build_pre_commit_config( languages: list[str], lint_languages: Iterable[str], pre_commit_config_location: Optional[Path] = None, - ) -> BuildConfigResult: + ) -> language.BuildConfigResult: """ Builds the final .pre-commit-config.yaml from all supported repo languages. Also returns any and all linter configuration data. :param languages: list of languages to get calculated configuration for. :param lint_languages: list of languages to add lint pre-commit hooks for. - :return: BuildConfigResult + :return: language.BuildConfigResult """ config_repos = [] existing_data = {} @@ -245,12 +158,13 @@ def build_pre_commit_config( try: data = yaml.safe_load(stream) existing_data = data or {} - config_repos += data["repos"] + config_repos += data["repos"] if data and data.get("repos") else [] + except yaml.YAMLError: self.echo.error( f"There was an issue parsing existing pre-commit-config.yaml." ) - return BuildConfigResult( + return language.BuildConfigResult( successful=False, languages_added=[], config_data={}, @@ -258,15 +172,17 @@ def build_pre_commit_config( linter_configs=linter_configs, ) - for language in config_languages: - include_linter = language in config_lint_languages - result = self.language_config.get_language_config(language, include_linter) + for config_language in config_languages: + include_linter = config_language in config_lint_languages + result = self.language_config.get_language_config( + config_language, include_linter + ) if result.config_data: - successful_languages.append(language) + successful_languages.append(config_language) ( linter_configs.append( LinterConfig( - language=language, + language=config_language, linter_data=result.linter_config.linter_data, ) ) @@ -275,10 +191,11 @@ def build_pre_commit_config( ) data = yaml.safe_load(result.config_data) config_repos += data["repos"] or [] + config = {**existing_data, "repos": config_repos} version = hash_config(yaml.dump(config)) - return BuildConfigResult( + return language.BuildConfigResult( successful=True if len(config_repos) > 0 else False, languages_added=successful_languages, config_data=config, @@ -286,10 +203,22 @@ def build_pre_commit_config( linter_configs=linter_configs, ) + def _create_repo(self, raw_repo: dict) -> Repo: + """ + Creates a repository containing pre-commit hooks from a raw dictionary object + :param raw_repo: dictionary containing repository data. + :return: repository containing pre-commit hooks + """ + return Repo( + repo=raw_repo.get("repo", "unknown"), + revision=raw_repo.get("rev", "unknown"), + hooks=[hook.get("id", "unknown") for hook in raw_repo.get("hooks", [])], + ) + def _write_pre_commit_configs( self, all_linter_configs: list[LinterConfig], - ) -> LinterConfigWriteResult: + ) -> language.LinterConfigWriteResult: """ Install any config files for given language to support any pre-commit commands. i.e. Javascript ESLint requires a .eslintrc file to sufficiently use plugins and allow @@ -306,16 +235,16 @@ def _write_pre_commit_configs( error_messages: list[str] = [] successful_languages: list[str] = [] - for config, language in linter_config_data: + for config, config_language in linter_config_data: try: with open(Path(SecureliConfig.FOLDER_PATH / config.filename), "w") as f: f.write(yaml.dump(config.settings)) - successful_languages.append(language) + successful_languages.append(config_language) except: error_messages.append( - f"Failed to write {config.filename} linter config file for {language}" + f"Failed to write {config.filename} linter config file for {config_language}" ) - return LinterConfigWriteResult( + return language.LinterConfigWriteResult( successful_languages=successful_languages, error_messages=error_messages ) diff --git a/secureli/consts/logging.py b/secureli/modules/observability/consts/logging.py similarity index 100% rename from secureli/consts/logging.py rename to secureli/modules/observability/consts/logging.py diff --git a/secureli/services/logging.py b/secureli/modules/observability/observability_services/logging.py similarity index 83% rename from secureli/services/logging.py rename to secureli/modules/observability/observability_services/logging.py index 1b223e85..dcdfad00 100644 --- a/secureli/services/logging.py +++ b/secureli/modules/observability/observability_services/logging.py @@ -6,12 +6,13 @@ from uuid import uuid4 import pydantic +from secureli.modules.shared.models.config import HookConfiguration +from secureli.modules.shared.models.logging import LogAction import secureli.repositories.secureli_config as SecureliConfig -from secureli.services.language_support import LanguageSupportService, HookConfiguration +from secureli.modules.language_analyzer import language_support from secureli.repositories.secureli_config import SecureliConfigRepository -from secureli.utilities.git_meta import current_branch_name, git_user_email, origin_url -from secureli.utilities.secureli_meta import secureli_version +from secureli.modules.shared import utilities def generate_unique_id() -> str: @@ -19,7 +20,7 @@ def generate_unique_id() -> str: A unique identifier representing the log entry, including various bits specific to the user and environment """ - origin_email_branch = f"{origin_url()}|{git_user_email()}|{current_branch_name()}" + origin_email_branch = f"{utilities.origin_url()}|{utilities.git_user_email()}|{utilities.current_branch_name()}" return f"{uuid4()}|{origin_email_branch}" @@ -30,16 +31,6 @@ class LogStatus(str, Enum): failure = "FAILURE" -class LogAction(str, Enum): - """Which action the log entry is associated with""" - - scan = "SCAN" - init = "INIT" - build = "_BUILD" - update = "UPDATE" - publish = "PUBLISH" # "PUBLISH" does not correspond to a CLI action/subcommand - - class LogFailure(pydantic.BaseModel): """An extendable structure for log failures""" @@ -51,9 +42,9 @@ class LogEntry(pydantic.BaseModel): id: str = generate_unique_id() timestamp: datetime = datetime.utcnow() - username: str = git_user_email() + username: str = utilities.git_user_email() machineid: str = platform.uname().node - secureli_version: str = secureli_version() + secureli_version: str = utilities.secureli_version() languages: Optional[list[str]] status: LogStatus action: LogAction @@ -68,7 +59,7 @@ class LoggingService: def __init__( self, - language_support: LanguageSupportService, + language_support: language_support.LanguageSupportService, secureli_config: SecureliConfigRepository, ): self.language_support = language_support @@ -133,7 +124,7 @@ def failure( def _log(self, log_entry: LogEntry): """Commit a log entry to the branch log file""" log_folder_path = Path(SecureliConfig.FOLDER_PATH / ".secureli/logs") - path_to_log = log_folder_path / f"{current_branch_name()}" + path_to_log = log_folder_path / f"{utilities.current_branch_name()}" # Do not simply mkdir the log folder path, in case the branch name contains # additional folder structure, like `bugfix/` or `feature/` diff --git a/secureli/modules/pii_scanner/pii_scanner.py b/secureli/modules/pii_scanner/pii_scanner.py new file mode 100644 index 00000000..b8c7a77e --- /dev/null +++ b/secureli/modules/pii_scanner/pii_scanner.py @@ -0,0 +1,214 @@ +from secureli.modules.shared.consts.pii import ( + DISABLE_PII_MARKER, + Format, + IGNORED_EXTENSIONS, + PII_CHECK, + RESULT_FORMAT, + SECURELI_GITHUB, +) +import os +import re +from typing import Optional +from pathlib import Path +import pydantic + +import secureli.modules.shared.models.scan as scan +from secureli.modules.shared.abstractions.echo import EchoAbstraction +from secureli.repositories.repo_files import RepoFilesRepository + + +class PiiResult(pydantic.BaseModel): + """ + An individual result of potential PII found + """ + + line_num: int + pii_key: str + + +class PiiScannerService: + """ + Scans the repo for potential PII + """ + + def __init__( + self, + repo_files: RepoFilesRepository, + echo: EchoAbstraction, + ): + self.repo_files = repo_files + self.echo = echo + + def scan_repo( + self, + folder_path: Path, + scan_mode: scan.ScanMode, + files: Optional[list[str]] = None, + ) -> scan.ScanResult: + """ + Scans the repo for potential PII + :param folder_path: The folder path to initialize the repo for + :param scan_mode: Whether to scan the staged files (i.e., the files about to be + committed) or the entire repository + :param files: A specified list of files to scan + :return: A ScanResult object with details of whether the scan succeeded and, if not, details of the failures + """ + + file_paths = self._get_files_list( + folder_path=folder_path, scan_mode=scan_mode, files=files + ) + current_line_num = 0 + pii_found: dict[str, list[PiiResult]] = {} + pii_found_files = set() + + for file_path in file_paths: + file_name = str(file_path) + try: + with open(file_path) as file: + for line in file: + current_line_num += 1 + for pii_key, pii_regex in PII_CHECK.items(): + if ( + re.search(pii_regex, line.lower()) + and not DISABLE_PII_MARKER in line + ): + if not file_name in pii_found: + pii_found[file_name] = [] + pii_found[file_name].append( + { + "line_num": current_line_num, + "pii_key": pii_key, + } + ) + pii_found_files.add(file_name) + current_line_num = 0 + + except Exception as e: + self.echo.print(f"Error PII scanning {file_name}: {e}") + scan_failures = self._generate_scan_failures(pii_found_files) + output = self._generate_scan_output(pii_found, not pii_found) + + return scan.ScanResult( + successful=not pii_found, + output=output, + failures=scan_failures, + ) + + def _file_extension_excluded(self, filename) -> bool: + _, file_extension = os.path.splitext(filename) + if file_extension in IGNORED_EXTENSIONS: + return True + + return False + + def _get_files_list( + self, + folder_path: Path, + scan_mode: scan.ScanMode, + files: Optional[list[str]] = None, + ) -> list[Path]: + """ + Gets the list of files to scan based on ScanMode and, if applicable, files provided in arguments + Note: Files cannot be specified for the `all-files` ScanMode. Also, if a provided file is not staged, + it will not be scanned + :param folder_path: The folder path to initialize the repo for + :param scan_mode: Whether to scan the staged files (i.e., the files about to be + committed) or the entire repository + :param files: A specified list of files to scan + :return: List of file names to be scanned + """ + file_paths: list[Path] = [] + + if scan_mode == scan.ScanMode.STAGED_ONLY: + file_paths = self.repo_files.list_staged_files(folder_path) + if files: + file_paths = list(filter(lambda file: file in file_paths, files)) + + if scan_mode == scan.ScanMode.ALL_FILES: + file_paths = self.repo_files.list_repo_files(folder_path) + + return list( + filter(lambda file: not self._file_extension_excluded(file), file_paths) + ) + + def _generate_scan_failures( + self, pii_found_files: set[str] + ) -> list[scan.ScanFailure]: + """ + Generates a list of ScanFailures for each file in which potential PII was found + :param pii_found_files: The set of files in which potential PII was found + :return: List of ScanFailures + """ + failures = [] + + for pii_found_file in pii_found_files: + failures.append( + scan.ScanFailure( + id="pii_scan", file=pii_found_file, repo=SECURELI_GITHUB + ) + ) + return failures + + def _generate_initial_output(self, success: bool) -> str: + """ + Generates the initial output of the PII scan, indicating whether the scan passed or failed + :param success: Whether the scan passed + :return: A string that will be used at the beginning of the output result + """ + CHECK_STR = "check for PII" + MAX_RESULT_LENGTH = ( + 82 # this aims to align with the results output by pre-commit hooks + ) + + result = ( + self._format_string("Passed", [Format.GREEN_BG]) + " " + if success + else self._format_string("Failed", [Format.RED_BG]) + "\n" + ) + length_of_dots = MAX_RESULT_LENGTH - len(CHECK_STR) - len(result) + final_msg = ( + "\n" + + self._format_string( + "Potential PII found!", [Format.BOLD_WEIGHT, Format.RED_TXT] + ) + if not success + else "" + ) + output = f"{CHECK_STR}{'.' * length_of_dots}{result}{final_msg}" + + return output + + def _generate_scan_output( + self, pii_found: dict[str, list[PiiResult]], success: bool + ) -> str: + """ + Generates the scan output of the PII scan, listing all the areas where potential PII was found + :param pii_found: The breakdown of what potential PII was found, and where + :param success: Whether the scan passed + :return: The final output result + """ + output = self._generate_initial_output(success) + for file, results in pii_found.items(): + output = ( + output + + "\n" + + self._format_string( + f"File: {file}", [Format.BOLD_WEIGHT, Format.PURPLE_TXT] + ) + ) + for result in results: + output = output + f"\n Line {result['line_num']}: {result['pii_key']}" + return output + "\n" + + def _format_string(self, str: str, formats: list[Format]) -> str: + """ + Applies formatting to a string + :param str: The string to format + :param formats: The formatting to apply to the string + :return: The formatted string + """ + + start = "".join(f"{RESULT_FORMAT[format]}" for format in formats) + end = f"{RESULT_FORMAT[Format.DEFAULT]}{RESULT_FORMAT[Format.REG_WEIGHT]}" + + return f"{start}{str}{end}" diff --git a/secureli/services/secureli_ignore.py b/secureli/modules/secureli_ignore.py similarity index 100% rename from secureli/services/secureli_ignore.py rename to secureli/modules/secureli_ignore.py diff --git a/secureli/utilities/__init__.py b/secureli/modules/shared/abstractions/__init__.py similarity index 100% rename from secureli/utilities/__init__.py rename to secureli/modules/shared/abstractions/__init__.py diff --git a/secureli/abstractions/echo.py b/secureli/modules/shared/abstractions/echo.py similarity index 90% rename from secureli/abstractions/echo.py rename to secureli/modules/shared/abstractions/echo.py index 78c3438f..93d34493 100644 --- a/secureli/abstractions/echo.py +++ b/secureli/modules/shared/abstractions/echo.py @@ -1,12 +1,9 @@ from abc import ABC, abstractmethod -from enum import Enum from typing import IO, Optional import sys import typer -from secureli.models.echo import Color - -from secureli.utilities.logging import EchoLevel +from secureli.modules.shared.models.echo import Color, Level class EchoAbstraction(ABC): @@ -18,15 +15,15 @@ class EchoAbstraction(ABC): """ def __init__(self, level: str): - self.print_enabled = level != EchoLevel.off - self.debug_enabled = level == EchoLevel.debug - self.info_enabled = level in [EchoLevel.debug, EchoLevel.info] - self.warn_enabled = level in [EchoLevel.debug, EchoLevel.info, EchoLevel.warn] + self.print_enabled = level != Level.off + self.debug_enabled = level == Level.debug + self.info_enabled = level in [Level.debug, Level.info] + self.warn_enabled = level in [Level.debug, Level.info, Level.warn] self.error_enabled = level in [ - EchoLevel.debug, - EchoLevel.info, - EchoLevel.warn, - EchoLevel.error, + Level.debug, + Level.info, + Level.warn, + Level.error, ] @abstractmethod diff --git a/secureli/abstractions/lexer_guesser.py b/secureli/modules/shared/abstractions/lexer_guesser.py similarity index 100% rename from secureli/abstractions/lexer_guesser.py rename to secureli/modules/shared/abstractions/lexer_guesser.py diff --git a/secureli/abstractions/pre_commit.py b/secureli/modules/shared/abstractions/pre_commit.py similarity index 95% rename from secureli/abstractions/pre_commit.py rename to secureli/modules/shared/abstractions/pre_commit.py index 65c2a122..5aa1eb10 100644 --- a/secureli/abstractions/pre_commit.py +++ b/secureli/modules/shared/abstractions/pre_commit.py @@ -7,7 +7,6 @@ # by implementing a dry-run option for the `autoupdate` command from pre_commit.commands.autoupdate import RevInfo as HookRepoRevInfo from typing import Optional -from secureli.abstractions.echo import EchoAbstraction import pydantic import re @@ -15,7 +14,8 @@ import subprocess import yaml -from secureli.repositories.settings import PreCommitSettings +from secureli.repositories.repo_settings import PreCommitSettings +from secureli.modules.shared.abstractions.echo import EchoAbstraction class InstallFailedError(Exception): @@ -344,12 +344,23 @@ def get_pre_commit_config_path(self, folder_path: Path) -> Path: f"Could not find pre-commit hooks in .secureli/{self.CONFIG_FILE_NAME}" ) + def get_pre_commit_config_path_is_correct(self, folder_path: Path) -> bool: + """Returns whether a pre-commit-config exists in a given folder path""" + preferred_pre_commit_config_location = ( + self.get_preferred_pre_commit_config_path(folder_path) + ) + pre_commit_config_path = folder_path / self.CONFIG_FILE_NAME + return ( + preferred_pre_commit_config_location.exists() + or pre_commit_config_path.exists() + and pre_commit_config_path == preferred_pre_commit_config_location + ) + def get_pre_commit_config(self, folder_path: Path): """ Gets the contents of the .pre-commit-config file and returns it as a dictionary :return: Dictionary containing the contents of the .pre-commit-config.yaml file """ - config_file_path: Path = self.get_pre_commit_config_path(folder_path) return self._read_pre_commit_config(config_file_path) @@ -367,7 +378,7 @@ def migrate_config_file(self, folder_path): Feel free to delete this method after an appropriate period of time (a few months?) """ existing_config_file_path = self.get_pre_commit_config_path(folder_path) - new_config_file_path = folder_path / ".secureli" / self.CONFIG_FILE_NAME + new_config_file_path = self.get_preferred_pre_commit_config_path(folder_path) self.echo.print( f"Moving {existing_config_file_path} to {new_config_file_path}..." ) diff --git a/secureli/modules/shared/abstractions/repo.py b/secureli/modules/shared/abstractions/repo.py new file mode 100644 index 00000000..6f548447 --- /dev/null +++ b/secureli/modules/shared/abstractions/repo.py @@ -0,0 +1,21 @@ +from abc import ABC, abstractmethod +import git + + +class RepoAbstraction(ABC): + """ + Abstracts the configuring and execution of git repo features. + """ + + @abstractmethod + def get_commit_diff(self) -> list[str]: + pass + + +class GitRepo(RepoAbstraction): + """ + Implementation and wrapper around git repo features + """ + + def get_commit_diff(self) -> list[str]: + return git.Repo().head.commit.diff() diff --git a/tests/abstractions/__init__.py b/secureli/modules/shared/consts/__init__.py similarity index 100% rename from tests/abstractions/__init__.py rename to secureli/modules/shared/consts/__init__.py diff --git a/secureli/modules/shared/consts/language.py b/secureli/modules/shared/consts/language.py new file mode 100644 index 00000000..5527489e --- /dev/null +++ b/secureli/modules/shared/consts/language.py @@ -0,0 +1,11 @@ +supported_languages = [ + "C#", + "Python", + "Java", + "Terraform", + "TypeScript", + "JavaScript", + "Go", + "Swift", + "Kotlin", +] diff --git a/secureli/modules/shared/consts/pii.py b/secureli/modules/shared/consts/pii.py new file mode 100644 index 00000000..be3fffe7 --- /dev/null +++ b/secureli/modules/shared/consts/pii.py @@ -0,0 +1,51 @@ +from enum import Enum + +PII_CHECK = { + "Email": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b", + "Social security number": r"(?!000|666|333)0*(?:[0-6][0-9][0-9]|[0-7][0-6][0-9]|[0-7][0-7][0-2])[- ](?!00)[0-9]{2}[- ](?!0000)[0-9]{4}", + "Phone number": r"[\+]?[(]?[0-9]{3}[)]?[-\s\.]?[0-9]{3}[-\s\.]?[0-9]{4,6}", +} + +IGNORED_EXTENSIONS = [ + ".md", + ".lock", + ".png", + ".jpg", + ".jpeg", + ".gif", + ".svg", + ".ico", + ".eot", + ".ttf", + ".woff", + ".css", +] + +SECURELI_GITHUB = "https://github.com/slalombuild/secureli" + + +class Format(str, Enum): + """ + Enum to use for formatting PII results + """ + + PURPLE_TXT = "purple_text" + RED_TXT = "red_text" + RED_BG = "red_background" + GREEN_BG = "green_background" + DEFAULT = "default_format" + BOLD_WEIGHT = "bold_weight" + REG_WEIGHT = "regular_weight" + + +RESULT_FORMAT = { + Format.DEFAULT: "\033[m", + Format.PURPLE_TXT: "\033[35m", + Format.RED_TXT: "\033[31m", + Format.GREEN_BG: "\033[42m", + Format.RED_BG: "\033[41m", + Format.BOLD_WEIGHT: "\033[1m", + Format.REG_WEIGHT: "\033[22m", +} + +DISABLE_PII_MARKER = "disable-pii-scan" diff --git a/tests/resources/__init__.py b/secureli/modules/shared/models/__init__.py similarity index 100% rename from tests/resources/__init__.py rename to secureli/modules/shared/models/__init__.py diff --git a/secureli/modules/shared/models/config.py b/secureli/modules/shared/models/config.py new file mode 100644 index 00000000..3d0d70a3 --- /dev/null +++ b/secureli/modules/shared/models/config.py @@ -0,0 +1,30 @@ +from typing import Any +import pydantic + + +class Repo(pydantic.BaseModel): + """A repository containing pre-commit hooks""" + + repo: str + revision: str + hooks: list[str] + + +class HookConfiguration(pydantic.BaseModel): + """A simplified pre-commit configuration representation for logging purposes""" + + repos: list[Repo] + + +class LinterConfigData(pydantic.BaseModel): + """ + Represents the structure of a linter config file + """ + + filename: str + settings: Any + + +class LinterConfig(pydantic.BaseModel): + language: str + linter_data: list[LinterConfigData] diff --git a/secureli/models/echo.py b/secureli/modules/shared/models/echo.py similarity index 60% rename from secureli/models/echo.py rename to secureli/modules/shared/models/echo.py index 71f6423c..f3cfe531 100644 --- a/secureli/models/echo.py +++ b/secureli/modules/shared/models/echo.py @@ -15,3 +15,17 @@ class Color(str, Enum): MAGENTA = "magenta" CYAN = "cyan" WHITE = "white" + + +class Level(str, Enum): + debug = "DEBUG" + info = "INFO" + warn = "WARN" + error = "ERROR" + off = "OFF" + + def __str__(self) -> str: + return self.value + + def __repr__(self) -> str: + return self.__str__() diff --git a/secureli/models/exit_codes.py b/secureli/modules/shared/models/exit_codes.py similarity index 100% rename from secureli/models/exit_codes.py rename to secureli/modules/shared/models/exit_codes.py diff --git a/secureli/modules/shared/models/install.py b/secureli/modules/shared/models/install.py new file mode 100644 index 00000000..6f91f293 --- /dev/null +++ b/secureli/modules/shared/models/install.py @@ -0,0 +1,30 @@ +from enum import Enum +from typing import Optional +from pathlib import Path + +import pydantic +from secureli.modules.shared.models.language import AnalyzeResult + +from secureli.repositories.secureli_config import SecureliConfig + + +class VerifyOutcome(str, Enum): + INSTALL_CANCELED = "install-canceled" + INSTALL_FAILED = "install-failed" + INSTALL_SUCCEEDED = "install-succeeded" + UPDATE_CANCELED = "update-canceled" + UPDATE_SUCCEEDED = "update-succeeded" + UPDATE_FAILED = "update-failed" + UP_TO_DATE = "up-to-date" + + +class VerifyResult(pydantic.BaseModel): + """ + The outcomes of performing verification. Actions can use these results + to decide whether to proceed with their post-initialization actions or not. + """ + + outcome: VerifyOutcome + config: Optional[SecureliConfig] = None + analyze_result: Optional[AnalyzeResult] = None + file_path: Optional[Path] = None diff --git a/secureli/modules/shared/models/language.py b/secureli/modules/shared/models/language.py new file mode 100644 index 00000000..3a115dca --- /dev/null +++ b/secureli/modules/shared/models/language.py @@ -0,0 +1,72 @@ +from pathlib import Path +from typing import Any, Optional +import pydantic + +from secureli.modules.shared.models.config import LinterConfig + + +class SkippedFile(pydantic.BaseModel): + """ + A file skipped by the analysis phase. + """ + + file_path: Path + error_message: str + + +class AnalyzeResult(pydantic.BaseModel): + """ + The result of the analysis phase. + """ + + language_proportions: dict[str, float] + skipped_files: list[SkippedFile] + + +class LanguageNotSupportedError(Exception): + """The given language was not supported by the PreCommitHooks abstraction""" + + pass + + +class LoadLinterConfigsResult(pydantic.BaseModel): + """Results from finding and loading any pre-commit configs for the language""" + + successful: bool + linter_data: list[Any] + + +class LanguagePreCommitResult(pydantic.BaseModel): + """ + A configuration model for a supported pre-commit-configurable language. + """ + + language: str + config_data: str + version: str + linter_config: LoadLinterConfigsResult + + +class LanguageMetadata(pydantic.BaseModel): + version: str + security_hook_id: Optional[str] + linter_config_write_errors: Optional[list[str]] = [] + + +class BuildConfigResult(pydantic.BaseModel): + """Result about building config for all laguages""" + + successful: bool + languages_added: list[str] + config_data: dict + linter_configs: list[LinterConfig] + version: str + + +class LinterConfigWriteResult(pydantic.BaseModel): + """ + Result from writing linter config files + """ + + successful_languages: list[str] + error_messages: list[str] diff --git a/secureli/modules/shared/models/logging.py b/secureli/modules/shared/models/logging.py new file mode 100644 index 00000000..1594b305 --- /dev/null +++ b/secureli/modules/shared/models/logging.py @@ -0,0 +1,11 @@ +from enum import Enum + + +class LogAction(str, Enum): + """Which action the log entry is associated with""" + + scan = "SCAN" + init = "INIT" + build = "_BUILD" + update = "UPDATE" + publish = "PUBLISH" # "PUBLISH" does not correspond to a CLI action/subcommand diff --git a/secureli/models/publish_results.py b/secureli/modules/shared/models/publish_results.py similarity index 80% rename from secureli/models/publish_results.py rename to secureli/modules/shared/models/publish_results.py index f4711f65..9d41de9a 100644 --- a/secureli/models/publish_results.py +++ b/secureli/modules/shared/models/publish_results.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from enum import Enum -from secureli.models.result import Result +from secureli.modules.shared.models.result import Result class PublishResultsOption(Enum): diff --git a/secureli/models/result.py b/secureli/modules/shared/models/result.py similarity index 100% rename from secureli/models/result.py rename to secureli/modules/shared/models/result.py diff --git a/secureli/modules/shared/models/scan.py b/secureli/modules/shared/models/scan.py new file mode 100644 index 00000000..d86e785a --- /dev/null +++ b/secureli/modules/shared/models/scan.py @@ -0,0 +1,41 @@ +from enum import Enum +from typing import Optional + +import pydantic + + +class ScanMode(str, Enum): + """ + Which scan mode to run as when we perform scanning. + """ + + STAGED_ONLY = "staged-only" + ALL_FILES = "all-files" + + +class ScanFailure(pydantic.BaseModel): + """ + Represents the details of a failed rule from a scan + """ + + repo: str + id: str + file: str + + +class ScanOutput(pydantic.BaseModel): + """ + Represents the parsed output from a scan + """ + + failures: list[ScanFailure] + + +class ScanResult(pydantic.BaseModel): + """ + The results of calling scan_repo + """ + + successful: bool + output: Optional[str] = None + failures: list[ScanFailure] diff --git a/secureli/modules/shared/resources/__init__.py b/secureli/modules/shared/resources/__init__.py new file mode 100644 index 00000000..3c4e7d07 --- /dev/null +++ b/secureli/modules/shared/resources/__init__.py @@ -0,0 +1 @@ +from secureli.modules.shared.resources.read_resource import read_resource diff --git a/secureli/resources/files/build.txt b/secureli/modules/shared/resources/files/build.txt similarity index 100% rename from secureli/resources/files/build.txt rename to secureli/modules/shared/resources/files/build.txt diff --git a/secureli/resources/files/configs/javascript.config.yaml b/secureli/modules/shared/resources/files/configs/javascript.config.yaml similarity index 100% rename from secureli/resources/files/configs/javascript.config.yaml rename to secureli/modules/shared/resources/files/configs/javascript.config.yaml diff --git a/secureli/resources/files/configs/typescript.config.yaml b/secureli/modules/shared/resources/files/configs/typescript.config.yaml similarity index 100% rename from secureli/resources/files/configs/typescript.config.yaml rename to secureli/modules/shared/resources/files/configs/typescript.config.yaml diff --git a/secureli/resources/files/epilog.md b/secureli/modules/shared/resources/files/epilog.md similarity index 100% rename from secureli/resources/files/epilog.md rename to secureli/modules/shared/resources/files/epilog.md diff --git a/secureli/resources/files/pre-commit/base/base-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/base/base-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/base/base-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/base/base-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/base/csharp-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/base/csharp-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/base/csharp-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/base/csharp-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/base/go-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/base/go-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/base/go-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/base/go-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/base/java-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/base/java-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/base/java-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/base/java-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/base/javascript-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/base/javascript-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/base/javascript-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/base/javascript-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/base/kotlin-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/base/kotlin-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/base/kotlin-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/base/kotlin-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/base/python-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/base/python-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/base/python-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/base/python-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/base/swift-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/base/swift-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/base/swift-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/base/swift-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/base/terraform-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/base/terraform-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/base/terraform-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/base/terraform-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/base/typescript-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/base/typescript-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/base/typescript-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/base/typescript-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/lint/base-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/lint/base-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/lint/base-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/lint/base-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/lint/csharp-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/lint/csharp-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/lint/csharp-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/lint/csharp-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/lint/go-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/lint/go-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/lint/go-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/lint/go-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/lint/java-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/lint/java-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/lint/java-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/lint/java-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/lint/javascript-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/lint/javascript-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/lint/javascript-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/lint/javascript-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/lint/kotlin-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/lint/kotlin-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/lint/kotlin-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/lint/kotlin-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/lint/python-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/lint/python-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/lint/python-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/lint/python-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/lint/swift-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/lint/swift-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/lint/swift-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/lint/swift-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/lint/terraform-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/lint/terraform-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/lint/terraform-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/lint/terraform-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/lint/typescript-pre-commit.yaml b/secureli/modules/shared/resources/files/pre-commit/lint/typescript-pre-commit.yaml similarity index 100% rename from secureli/resources/files/pre-commit/lint/typescript-pre-commit.yaml rename to secureli/modules/shared/resources/files/pre-commit/lint/typescript-pre-commit.yaml diff --git a/secureli/resources/files/pre-commit/secrets_detecting_repos.yaml b/secureli/modules/shared/resources/files/pre-commit/secrets_detecting_repos.yaml similarity index 100% rename from secureli/resources/files/pre-commit/secrets_detecting_repos.yaml rename to secureli/modules/shared/resources/files/pre-commit/secrets_detecting_repos.yaml diff --git a/secureli/resources/read_resource.py b/secureli/modules/shared/resources/read_resource.py similarity index 100% rename from secureli/resources/read_resource.py rename to secureli/modules/shared/resources/read_resource.py diff --git a/secureli/resources/slugify.py b/secureli/modules/shared/resources/slugify.py similarity index 100% rename from secureli/resources/slugify.py rename to secureli/modules/shared/resources/slugify.py diff --git a/secureli/modules/shared/utilities.py b/secureli/modules/shared/utilities.py new file mode 100644 index 00000000..ac04a6d2 --- /dev/null +++ b/secureli/modules/shared/utilities.py @@ -0,0 +1,176 @@ +import configparser +import hashlib +import os +import requests +import subprocess + +from collections import Counter +from importlib.metadata import version +from typing import Optional + +from secureli.modules.observability.consts import logging +from secureli.modules.shared.models.publish_results import PublishLogResult +from secureli.modules.shared.models.result import Result +from secureli.modules.shared.models.scan import ScanFailure, ScanResult +from secureli.settings import Settings + + +def combine_patterns(patterns: list[str]) -> Optional[str]: + """ + Combines a set of patterns from pathspec into a single combined regex + suitable for use in circumstances that require a broad matching, such as + re.find_all or a pre-commit exclusion pattern. + + Calling this with an empty set of patterns will return None + :param patterns: The pattern strings provided by pathspec to combine + :return: a combined pattern containing the one or more patterns supplied, or None + if no patterns were supplied + """ + if not patterns: + return None + + if len(patterns) == 1: + return patterns[0] + + # Don't mutate the input + ignored_file_patterns = patterns.copy() + + # Quick sanitization process. Ultimately we combine all the regexes into a single + # entry, and all patterns are generated with a capture group with the same ID, which + # is invalid. We need to make sure every capture group is unique to combine them + # into a single expression. This is not my favorite. + for i in range(0, len(ignored_file_patterns)): + ignored_file_patterns[i] = str.replace( + ignored_file_patterns[i], "ps_d", f"ps_d{i}" + ) + combined_patterns = str.join("|", ignored_file_patterns) + combined_ignore_pattern = f"^({combined_patterns})$" + return combined_ignore_pattern + + +def convert_failures_to_failure_count(failure_list: list[ScanFailure]): + """ + Convert a list of Failure ids to a list of individual failure count with underscore naming convention + :param failure_list: a list of Failure Object + """ + list_of_failure_ids = [] + + for failure in failure_list: + failure_id_underscore = failure.id.replace("-", "_") + list_of_failure_ids.append(failure_id_underscore) + + failure_count_list = Counter(list_of_failure_ids) + return failure_count_list + + +def current_branch_name() -> str: + """Leverage the git HEAD file to determine the current branch name""" + try: + with open(".git/HEAD", "r") as f: + content = f.readlines() + for line in content: + if line[0:4] == "ref:": + return line.partition("refs/heads/")[2].strip() + except IOError: + return "UNKNOWN" + + +def format_sentence_list(items: list[str]) -> str: + """ + Formats a list of string values to a comma separated + string for use in a sentence structure. + :param items: list of strings to join + :return string of joined values as a sentence comma list + i.e. "x, y, and z" or "x and y" + """ + if not items: + return "" + elif len(items) == 1: + return items[0] + and_separator = ", and " if len(items) > 2 else " and " + return ", ".join(items[:-1]) + and_separator + items[-1] + + +def git_user_email() -> str: + """Leverage the command prompt to derive the user's email address""" + args = ["git", "config", "user.email"] + completed_process = subprocess.run(args, stdout=subprocess.PIPE) + output = completed_process.stdout.decode("utf8").strip() + return output + + +def hash_config(config: str) -> str: + """ + Creates an MD5 hash from a config string + :return: A hash string + """ + config_hash = hashlib.md5(config.encode("utf8"), usedforsecurity=False).hexdigest() + + return config_hash + + +def origin_url() -> str: + """Leverage the git config file to determine the remote origin URL""" + git_config_parser = configparser.ConfigParser() + git_config_parser.read(".git/config") + return ( + git_config_parser['remote "origin"'].get("url", "UNKNOWN", raw=True) + if git_config_parser.has_section('remote "origin"') + else "UNKNOWN" + ) + + +def post_log(log_data: str, settings: Settings) -> PublishLogResult: + """ + Send a log through http post + :param log_data: a string to be sent to backend instrumentation + """ + + api_endpoint = ( + os.getenv(logging.TELEMETRY_ENDPOINT_ENV_VAR_NAME) or settings.telemetry.api_url + ) + api_key = os.getenv(logging.TELEMETRY_KEY_ENV_VAR_NAME) + + if not api_endpoint or not api_key: + return PublishLogResult( + result=Result.FAILURE, + result_message=f"{logging.TELEMETRY_ENDPOINT_ENV_VAR_NAME} or {logging.TELEMETRY_KEY_ENV_VAR_NAME} not found in environment variables", + ) + + try: + result = requests.post( + url=api_endpoint, headers={"Api-Key": api_key}, data=log_data + ) + except Exception as e: + return PublishLogResult( + result=Result.FAILURE, + result_message=f'Error posting log to {api_endpoint}: "{e}"', + ) + + return PublishLogResult(result=Result.SUCCESS, result_message=result.text) + + +def secureli_version() -> str: + """Leverage package resources to determine the current version of secureli""" + return version("secureli") + + +def merge_scan_results(results: list[ScanResult]): + """ + Creates a single ScanResult from multiple ScanResults + :param results: The list of ScanResults to merge + :return A single ScanResult + """ + final_successful = True + final_output = "" + final_failures: list[ScanFailure] = [] + + for result in results: + if result: + final_successful = final_successful and result.successful + final_output = final_output + (result.output or "") + "\n" + final_failures = final_failures + result.failures + + return ScanResult( + successful=final_successful, output=final_output, failures=final_failures + ) diff --git a/secureli/repositories/repo_files.py b/secureli/repositories/repo_files.py index 1f0119c3..6423a078 100644 --- a/secureli/repositories/repo_files.py +++ b/secureli/repositories/repo_files.py @@ -1,8 +1,9 @@ from pathlib import Path import re +import subprocess import chardet -from secureli.utilities.patterns import combine_patterns +from secureli.modules.shared.utilities import combine_patterns class BinaryFileError(ValueError): @@ -37,10 +38,7 @@ def list_repo_files(self, folder_path: Path) -> list[Path]: :raises ValueError: The specified path does not exist or is not a git repo :return: The visible files within the specified repo as a list of Path objects """ - git_path = folder_path / ".git" - - if not git_path.exists() or not git_path.is_dir(): - raise ValueError("The current folder is not a Git repository!") + self._confirm_is_git_repo(folder_path) file_paths = [ f @@ -117,3 +115,34 @@ def load_file(self, file_path: Path) -> str: pass raise ValueError(f"An unknown error occurred loading the file from {file_path}") + + def list_staged_files(self, folder_path: Path) -> list[Path]: + """ + Lists the file paths of all staged files in a repository, or raises ValueError if the provided path + is not a git repo + :param folder_path: The path to a git repo containing files + :raises ValueError: The specified path does not exist or is not a git repo + :return: The list of staged file names + """ + self._confirm_is_git_repo(folder_path) + git_diff_result = ( + subprocess.run( + ["git", "diff", "--staged", "--name-only"], + stdout=subprocess.PIPE, + cwd=folder_path, + ) + .stdout.decode("utf8") + .split("\n") + ) + return git_diff_result[0:-1] + + def _confirm_is_git_repo(self, folder_path: Path): + """ + Confirms a provided path is a git repo, and if not raises a ValueError + :param folder_path: The path to a git repo containing files + :raises ValueError: The specified path does not exist or is not a git repo + """ + git_path = folder_path / ".git" + + if not git_path.exists() or not git_path.is_dir(): + raise ValueError("The current folder is not a Git repository!") diff --git a/secureli/repositories/settings.py b/secureli/repositories/repo_settings.py similarity index 96% rename from secureli/repositories/settings.py rename to secureli/repositories/repo_settings.py index 3717a70e..45075ea1 100644 --- a/secureli/repositories/settings.py +++ b/secureli/repositories/repo_settings.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Optional from pydantic import BaseModel, BaseSettings, Field -from secureli.utilities.logging import EchoLevel +from secureli.modules.shared.models.echo import Level import yaml @@ -74,7 +74,7 @@ class EchoSettings(BaseSettings): Settings that affect how seCureLI provides information to the user. """ - level: EchoLevel = Field(default=EchoLevel.warn) + level: Level = Field(default=Level.warn) class LanguageSupportSettings(BaseSettings): @@ -157,7 +157,7 @@ def save(self, settings: SecureliFile): key: value for (key, value) in settings.dict().items() if value is not None } - # Converts EchoLevel to string + # Converts echo Level to string if settings_dict.get("echo"): settings_dict["echo"]["level"] = "{}".format(settings_dict["echo"]["level"]) diff --git a/secureli/resources/__init__.py b/secureli/resources/__init__.py deleted file mode 100644 index 491b4888..00000000 --- a/secureli/resources/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from secureli.resources.read_resource import read_resource diff --git a/secureli/settings.py b/secureli/settings.py index 60490769..ca257c8a 100644 --- a/secureli/settings.py +++ b/secureli/settings.py @@ -5,12 +5,7 @@ import yaml import secureli.repositories.secureli_config as SecureliConfig -from secureli.repositories.settings import ( - RepoFilesSettings, - EchoSettings, - LanguageSupportSettings, - TelemetrySettings, -) +from secureli.repositories import repo_settings def secureli_yaml_settings( @@ -39,10 +34,12 @@ class Settings(pydantic.BaseSettings): the .secureli.yaml file. """ - repo_files: RepoFilesSettings = RepoFilesSettings() - echo: EchoSettings = EchoSettings() - language_support: LanguageSupportSettings = LanguageSupportSettings() - telemetry: TelemetrySettings = TelemetrySettings() + repo_files: repo_settings.RepoFilesSettings = repo_settings.RepoFilesSettings() + echo: repo_settings.EchoSettings = repo_settings.EchoSettings() + language_support: repo_settings.LanguageSupportSettings = ( + repo_settings.LanguageSupportSettings() + ) + telemetry: repo_settings.TelemetrySettings = repo_settings.TelemetrySettings() class Config: env_file_encoding = "utf-8" diff --git a/secureli/utilities/formatter.py b/secureli/utilities/formatter.py deleted file mode 100644 index be7f9f71..00000000 --- a/secureli/utilities/formatter.py +++ /dev/null @@ -1,14 +0,0 @@ -def format_sentence_list(items: list[str]) -> str: - """ - Formats a list of string values to a comma separated - string for use in a sentence structure. - :param items: list of strings to join - :return string of joined values as a sentence comma list - i.e. "x, y, and z" or "x and y" - """ - if not items: - return "" - elif len(items) == 1: - return items[0] - and_separator = ", and " if len(items) > 2 else " and " - return ", ".join(items[:-1]) + and_separator + items[-1] diff --git a/secureli/utilities/git_meta.py b/secureli/utilities/git_meta.py deleted file mode 100644 index e15bc0c7..00000000 --- a/secureli/utilities/git_meta.py +++ /dev/null @@ -1,33 +0,0 @@ -import subprocess -import configparser - - -def git_user_email() -> str: - """Leverage the command prompt to derive the user's email address""" - args = ["git", "config", "user.email"] - completed_process = subprocess.run(args, stdout=subprocess.PIPE) - output = completed_process.stdout.decode("utf8").strip() - return output - - -def origin_url() -> str: - """Leverage the git config file to determine the remote origin URL""" - git_config_parser = configparser.ConfigParser() - git_config_parser.read(".git/config") - return ( - git_config_parser['remote "origin"'].get("url", "UNKNOWN", raw=True) - if git_config_parser.has_section('remote "origin"') - else "UNKNOWN" - ) - - -def current_branch_name() -> str: - """Leverage the git HEAD file to determine the current branch name""" - try: - with open(".git/HEAD", "r") as f: - content = f.readlines() - for line in content: - if line[0:4] == "ref:": - return line.partition("refs/heads/")[2].strip() - except IOError: - return "UNKNOWN" diff --git a/secureli/utilities/hash.py b/secureli/utilities/hash.py deleted file mode 100644 index 1af53f6d..00000000 --- a/secureli/utilities/hash.py +++ /dev/null @@ -1,11 +0,0 @@ -import hashlib - - -def hash_config(config: str) -> str: - """ - Creates an MD5 hash from a config string - :return: A hash string - """ - config_hash = hashlib.md5(config.encode("utf8"), usedforsecurity=False).hexdigest() - - return config_hash diff --git a/secureli/utilities/logging.py b/secureli/utilities/logging.py deleted file mode 100644 index 49642cae..00000000 --- a/secureli/utilities/logging.py +++ /dev/null @@ -1,15 +0,0 @@ -from enum import Enum - - -class EchoLevel(str, Enum): - debug = "DEBUG" - info = "INFO" - warn = "WARN" - error = "ERROR" - off = "OFF" - - def __str__(self) -> str: - return self.value - - def __repr__(self) -> str: - return self.__str__() diff --git a/secureli/utilities/patterns.py b/secureli/utilities/patterns.py deleted file mode 100644 index 0a845e39..00000000 --- a/secureli/utilities/patterns.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import Optional - - -def combine_patterns(patterns: list[str]) -> Optional[str]: - """ - Combines a set of patterns from pathspec into a single combined regex - suitable for use in circumstances that require a broad matching, such as - re.find_all or a pre-commit exclusion pattern. - - Calling this with an empty set of patterns will return None - :param patterns: The pattern strings provided by pathspec to combine - :return: a combined pattern containing the one or more patterns supplied, or None - if no patterns were supplied - """ - if not patterns: - return None - - if len(patterns) == 1: - return patterns[0] - - # Don't mutate the input - ignored_file_patterns = patterns.copy() - - # Quick sanitization process. Ultimately we combine all the regexes into a single - # entry, and all patterns are generated with a capture group with the same ID, which - # is invalid. We need to make sure every capture group is unique to combine them - # into a single expression. This is not my favorite. - for i in range(0, len(ignored_file_patterns)): - ignored_file_patterns[i] = str.replace( - ignored_file_patterns[i], "ps_d", f"ps_d{i}" - ) - combined_patterns = str.join("|", ignored_file_patterns) - combined_ignore_pattern = f"^({combined_patterns})$" - return combined_ignore_pattern diff --git a/secureli/utilities/secureli_meta.py b/secureli/utilities/secureli_meta.py deleted file mode 100644 index 6478683e..00000000 --- a/secureli/utilities/secureli_meta.py +++ /dev/null @@ -1,6 +0,0 @@ -from importlib.metadata import version - - -def secureli_version() -> str: - """Leverage package resources to determine the current version of secureli""" - return version("secureli") diff --git a/secureli/utilities/usage_stats.py b/secureli/utilities/usage_stats.py deleted file mode 100644 index d6b3f842..00000000 --- a/secureli/utilities/usage_stats.py +++ /dev/null @@ -1,58 +0,0 @@ -import requests -import os -from secureli.consts.logging import ( - TELEMETRY_ENDPOINT_ENV_VAR_NAME, - TELEMETRY_KEY_ENV_VAR_NAME, -) -from secureli.models.publish_results import PublishLogResult -from secureli.models.result import Result - -from secureli.services.scanner import Failure -from collections import Counter - -from secureli.settings import Settings - - -def convert_failures_to_failure_count(failure_list: list[Failure]): - """ - Convert a list of Failure ids to a list of individual failure count with underscore naming convention - :param failure_list: a list of Failure Object - """ - list_of_failure_ids = [] - - for failure in failure_list: - failure_id_underscore = failure.id.replace("-", "_") - list_of_failure_ids.append(failure_id_underscore) - - failure_count_list = Counter(list_of_failure_ids) - return failure_count_list - - -def post_log(log_data: str, settings: Settings) -> PublishLogResult: - """ - Send a log through http post - :param log_data: a string to be sent to backend instrumentation - """ - - api_endpoint = ( - os.getenv(TELEMETRY_ENDPOINT_ENV_VAR_NAME) or settings.telemetry.api_url - ) - api_key = os.getenv(TELEMETRY_KEY_ENV_VAR_NAME) - - if not api_endpoint or not api_key: - return PublishLogResult( - result=Result.FAILURE, - result_message=f"{TELEMETRY_ENDPOINT_ENV_VAR_NAME} or {TELEMETRY_KEY_ENV_VAR_NAME} not found in environment variables", - ) - - try: - result = requests.post( - url=api_endpoint, headers={"Api-Key": api_key}, data=log_data - ) - except Exception as e: - return PublishLogResult( - result=Result.FAILURE, - result_message=f'Error posting log to {api_endpoint}: "{e}"', - ) - - return PublishLogResult(result=Result.SUCCESS, result_message=result.text) diff --git a/tests/actions/conftest.py b/tests/actions/conftest.py index dd75a825..208458bb 100644 --- a/tests/actions/conftest.py +++ b/tests/actions/conftest.py @@ -2,8 +2,7 @@ import pytest -from secureli.services.language_analyzer import AnalyzeResult -from secureli.services.language_support import LanguageMetadata +from secureli.modules.shared.models.language import AnalyzeResult, LanguageMetadata # Register generic mocks you'd like available for every test. diff --git a/tests/actions/test_action.py b/tests/actions/test_action.py index 6046639c..83d575de 100644 --- a/tests/actions/test_action.py +++ b/tests/actions/test_action.py @@ -1,25 +1,25 @@ from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, patch, mock_open import pytest -from secureli.abstractions.pre_commit import InstallResult - -from secureli.actions.action import Action, ActionDependencies, VerifyOutcome -from secureli.consts.logging import TELEMETRY_DEFAULT_ENDPOINT -from secureli.models.echo import Color +from secureli.modules.shared.abstractions.pre_commit import InstallResult + +from secureli.actions.action import Action, ActionDependencies +from secureli.modules.observability.consts.logging import TELEMETRY_DEFAULT_ENDPOINT +from secureli.modules.shared.models.echo import Color +from secureli.modules.shared.models.install import VerifyOutcome +from secureli.modules.shared.models import language +from secureli.modules.shared.models.scan import ScanFailure, ScanResult from secureli.repositories.secureli_config import SecureliConfig, VerifyConfigOutcome -from secureli.services.language_analyzer import AnalyzeResult, SkippedFile -from secureli.services.language_support import LanguageMetadata -from secureli.services.scanner import ScanResult, Failure -from secureli.services.updater import UpdateResult +from secureli.modules.core.core_services.updater import UpdateResult test_folder_path = Path("does-not-matter") @pytest.fixture() -def mock_scanner() -> MagicMock: - mock_scanner = MagicMock() - return mock_scanner +def mock_hooks_scanner() -> MagicMock: + mock_hooks_scanner = MagicMock() + return mock_hooks_scanner @pytest.fixture() @@ -33,7 +33,7 @@ def action_deps( mock_echo: MagicMock, mock_language_analyzer: MagicMock, mock_language_support: MagicMock, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_secureli_config: MagicMock, mock_updater: MagicMock, mock_settings: MagicMock, @@ -42,7 +42,7 @@ def action_deps( mock_echo, mock_language_analyzer, mock_language_support, - mock_scanner, + mock_hooks_scanner, mock_secureli_config, mock_settings, mock_updater, @@ -59,11 +59,11 @@ def test_that_initialize_repo_raises_value_error_without_any_supported_languages mock_language_analyzer: MagicMock, mock_echo: MagicMock, ): - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={}, skipped_files=[] ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) mock_echo.error.assert_called_with( "seCureLI could not be installed due to an error: No supported languages found in current repository" @@ -75,7 +75,7 @@ def test_that_initialize_repo_install_flow_selects_both_languages( mock_language_analyzer: MagicMock, mock_echo: MagicMock, ): - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={ "RadLang": 0.75, "CoolLang": 0.25, @@ -83,7 +83,7 @@ def test_that_initialize_repo_install_flow_selects_both_languages( skipped_files=[], ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) mock_echo.print.assert_called_with( "seCureLI has been installed successfully for the following language(s): RadLang and CoolLang.\n", @@ -95,9 +95,9 @@ def test_that_initialize_repo_install_flow_selects_both_languages( def test_that_initialize_repo_install_flow_performs_security_analysis( action: Action, mock_language_analyzer: MagicMock, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, ): - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={ "RadLang": 0.75, "BadLang": 0.25, @@ -105,20 +105,20 @@ def test_that_initialize_repo_install_flow_performs_security_analysis( skipped_files=[], ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) - mock_scanner.scan_repo.assert_called_once() + mock_hooks_scanner.scan_repo.assert_called_once() def test_that_initialize_repo_install_flow_displays_security_analysis_results( - action: Action, action_deps: MagicMock, mock_scanner: MagicMock + action: Action, action_deps: MagicMock, mock_hooks_scanner: MagicMock ): - mock_scanner.scan_repo.return_value = ScanResult( + mock_hooks_scanner.scan_repo.return_value = ScanResult( successful=False, output="Detect secrets...Failed", - failures=[Failure(repo="repo", id="id", file="file")], + failures=[ScanFailure(repo="repo", id="id", file="file")], ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) action_deps.echo.print.assert_any_call("Detect secrets...Failed") @@ -126,23 +126,23 @@ def test_that_initialize_repo_install_flow_displays_security_analysis_results( def test_that_initialize_repo_install_flow_skips_security_analysis_if_unavailable( action: Action, mock_language_analyzer: MagicMock, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_language_support: MagicMock, ): - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={ "RadLang": 0.75, "BadLang": 0.25, }, skipped_files=[], ) - mock_language_support.apply_support.return_value = LanguageMetadata( + mock_language_support.apply_support.return_value = language.LanguageMetadata( version="abc123", security_hook_id=None ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) - mock_scanner.scan_repo.assert_not_called() + mock_hooks_scanner.scan_repo.assert_not_called() def test_that_initialize_repo_install_flow_warns_about_skipped_files( @@ -151,17 +151,17 @@ def test_that_initialize_repo_install_flow_warns_about_skipped_files( mock_echo: MagicMock, mock_updater: MagicMock, ): - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={ "RadLang": 0.75, "BadLang": 0.25, }, skipped_files=[ - SkippedFile( + language.SkippedFile( file_path=Path("./file.wacky-extension"), error_message="What a wacky extension!", ), - SkippedFile( + language.SkippedFile( file_path=Path("./file2.huge"), error_message="What a huge file!" ), ], @@ -171,7 +171,7 @@ def test_that_initialize_repo_install_flow_warns_about_skipped_files( successful=True, backup_hook_path=None ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) assert ( mock_echo.warning.call_count == 3 @@ -181,12 +181,18 @@ def test_that_initialize_repo_install_flow_warns_about_skipped_files( def test_that_initialize_repo_can_be_canceled( action: Action, mock_echo: MagicMock, + mock_hooks_scanner: MagicMock, ): mock_echo.confirm.return_value = False + mock_hooks_scanner.pre_commit.get_pre_commit_config_path_is_correct.return_value = ( + True + ) + with (patch.object(Path, "exists", return_value=True),): + action.verify_install( + test_folder_path, reset=True, always_yes=False, files=None + ) - action.verify_install(test_folder_path, reset=True, always_yes=False) - - mock_echo.error.assert_called_with("User canceled install process") + mock_echo.error.assert_called_with("User canceled install process") def test_that_initialize_repo_selects_previously_selected_language( @@ -196,7 +202,7 @@ def test_that_initialize_repo_selects_previously_selected_language( mock_echo: MagicMock, mock_language_analyzer: MagicMock, ): - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={"PreviousLang": 1.0}, skipped_files=[], ) @@ -205,7 +211,7 @@ def test_that_initialize_repo_selects_previously_selected_language( ) mock_language_support.version_for_language.return_value = "abc123" - action.verify_install(test_folder_path, reset=False, always_yes=True) + action.verify_install(test_folder_path, reset=False, always_yes=True, files=None) mock_echo.print.assert_called_with( "seCureLI is installed and up-to-date for the following language(s): PreviousLang" @@ -223,7 +229,7 @@ def test_that_initialize_repo_prompts_to_upgrade_config_if_old_schema( mock_language_support.version_for_language.return_value = "xyz987" mock_echo.confirm.return_value = False - action.verify_install(test_folder_path, reset=False, always_yes=False) + action.verify_install(test_folder_path, reset=False, always_yes=False, files=None) mock_echo.error.assert_called_with("seCureLI could not be verified.") @@ -234,7 +240,7 @@ def test_that_initialize_repo_updates_repo_config_if_old_schema( mock_language_support: MagicMock, mock_language_analyzer: MagicMock, ): - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={"PreviousLang": 1.0}, skipped_files=[], ) @@ -250,7 +256,9 @@ def test_that_initialize_repo_updates_repo_config_if_old_schema( mock_language_support.version_for_language.return_value = "abc123" - result = action.verify_install(test_folder_path, reset=False, always_yes=True) + result = action.verify_install( + test_folder_path, reset=False, always_yes=True, files=None + ) assert result.outcome == VerifyOutcome.UP_TO_DATE @@ -265,7 +273,7 @@ def test_that_initialize_repo_reports_errors_when_schema_update_fails( mock_secureli_config.update.side_effect = Exception - action.verify_install(test_folder_path, reset=False, always_yes=True) + action.verify_install(test_folder_path, reset=False, always_yes=True, files=None) mock_echo.error.assert_called_with("seCureLI could not be verified.") @@ -280,7 +288,7 @@ def test_that_initialize_repo_is_aborted_by_the_user_if_the_process_is_canceled( mock_echo.confirm.return_value = False mock_secureli_config.load.return_value = SecureliConfig() # fresh config - action.verify_install(test_folder_path, reset=False, always_yes=False) + action.verify_install(test_folder_path, reset=False, always_yes=False, files=None) mock_echo.error.assert_called_with("User canceled install process") @@ -296,11 +304,13 @@ def test_that_initialize_repo_returns_up_to_date_if_the_process_is_canceled_on_e mock_secureli_config.load.return_value = SecureliConfig( languages=["RadLang"], version_installed="abc123" ) - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={"RadLang": 0.5, "CoolLang": 0.5}, skipped_files=[] ) - result = action.verify_install(test_folder_path, reset=False, always_yes=False) + result = action.verify_install( + test_folder_path, reset=False, always_yes=False, files=None + ) assert result.outcome == VerifyOutcome.UP_TO_DATE @@ -312,7 +322,7 @@ def test_that_initialize_repo_prints_warnings_for_failed_linter_config_writes( ): config_write_error = "Failed to write config file for RadLang" - mock_language_support.apply_support.return_value = LanguageMetadata( + mock_language_support.apply_support.return_value = language.LanguageMetadata( version="abc123", security_hook_id="test_hook_id", linter_config_write_errors=[config_write_error], @@ -322,7 +332,7 @@ def test_that_initialize_repo_prints_warnings_for_failed_linter_config_writes( successful=True, backup_hook_path=None ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) mock_echo.warning.assert_called_once_with(config_write_error) @@ -336,12 +346,12 @@ def test_that_verify_install_returns_failed_result_on_new_install_language_not_s languages=[], version_installed=None ) - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={}, skipped_files=[] ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=False + test_folder_path, reset=False, always_yes=False, files=None ) assert verify_result.outcome == VerifyOutcome.INSTALL_FAILED @@ -356,12 +366,12 @@ def test_that_verify_install_returns_up_to_date_result_on_existing_install_langu languages=["RadLang"], version_installed="abc123" ) - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={}, skipped_files=[] ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=False + test_folder_path, reset=False, always_yes=False, files=None ) assert verify_result.outcome == VerifyOutcome.UP_TO_DATE @@ -376,12 +386,12 @@ def test_that_verify_install_returns_up_to_date_result_on_existing_install_no_ne languages=["RadLang"], version_installed="abc123" ) - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={"RadLang": 1.0}, skipped_files=[] ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=False + test_folder_path, reset=False, always_yes=False, files=None ) assert verify_result.outcome == VerifyOutcome.UP_TO_DATE @@ -396,54 +406,63 @@ def test_that_verify_install_returns_success_result_newly_detected_language_inst languages=["RadLang"], version_installed="abc123" ) - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={"RadLang": 0.5, "CoolLang": 0.5}, skipped_files=[] ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=True + test_folder_path, reset=False, always_yes=True, files=None ) assert verify_result.outcome == VerifyOutcome.INSTALL_SUCCEEDED -def test_that_verify_install_returns_failure_result_without_re_commit_config_file_path( +def test_that_verify_install_returns_failure_result_without_pre_commit_config_file_path( action: Action, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_echo: MagicMock, ): with (patch.object(Path, "exists", return_value=False),): - mock_scanner.pre_commit.get_preferred_pre_commit_config_path.return_value = ( + mock_hooks_scanner.pre_commit.get_preferred_pre_commit_config_path.return_value = ( test_folder_path / ".secureli" / ".pre-commit-config.yaml" ) - mock_scanner.pre_commit.migrate_config_file.side_effect = Exception("ERROR") + mock_hooks_scanner.pre_commit.get_pre_commit_config_path_is_correct.return_value = ( + False + ) + mock_hooks_scanner.pre_commit.migrate_config_file.side_effect = Exception( + "ERROR" + ) + verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=True + test_folder_path, reset=False, always_yes=True, files=None ) mock_echo.error.assert_called_once_with( - "seCureLI pre-commit-config.yaml could not be updated." + "seCureLI .pre-commit-config.yaml could not be moved." ) assert verify_result.outcome == VerifyOutcome.UPDATE_FAILED def test_that_verify_install_continues_after_pre_commit_config_file_moved( action: Action, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_echo: MagicMock, ): - with (patch.object(Path, "exists", return_value=False),): - mock_scanner.pre_commit.get_preferred_pre_commit_config_path.return_value = ( + with (patch.object(Path, "exists", return_value=True),): + mock_hooks_scanner.pre_commit.get_preferred_pre_commit_config_path.return_value = ( test_folder_path / ".secureli" / ".pre-commit-config.yaml" ) + mock_hooks_scanner.pre_commit.get_pre_commit_config_path_is_correct.return_value = ( + False + ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=True + test_folder_path, reset=False, always_yes=True, files=None ) assert verify_result.outcome == VerifyOutcome.INSTALL_SUCCEEDED def test_that_update_secureli_pre_commit_config_location_moves_file( action: Action, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_echo: MagicMock, ): update_file_location = test_folder_path / ".secureli" / ".pre-commit-config.yaml" @@ -451,30 +470,33 @@ def test_that_update_secureli_pre_commit_config_location_moves_file( update_file_location, True ) mock_echo.print.assert_called_once_with( - "seCureLI's .pre-commit-config.yaml is in a deprecated location." + "The .pre-commit-config.yaml is in a deprecated location." + ) + mock_hooks_scanner.pre_commit.migrate_config_file.assert_called_with( + update_file_location ) - mock_scanner.pre_commit.migrate_config_file.assert_called_with(update_file_location) assert update_result.outcome == VerifyOutcome.UPDATE_SUCCEEDED + # assert update_result.file_path == update_file_location def test_that_update_secureli_pre_commit_config_fails_on_exception( action: Action, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, ): - with pytest.raises(Exception): - mock_scanner.pre_commit.get_preferred_pre_commit_config_path.return_value = ( - test_folder_path / ".secureli" / ".pre-commit-config.yaml" - ) - update_result = action._update_secureli_pre_commit_config_location( - test_folder_path, True - ) - assert update_result.outcome == VerifyOutcome.UPDATE_FAILED + mock_hooks_scanner.pre_commit.migrate_config_file.side_effect = Exception("ERROR") + mock_hooks_scanner.pre_commit.get_preferred_pre_commit_config_path.return_value = ( + test_folder_path / ".secureli" / ".pre-commit-config.yaml" + ) + update_result = action._update_secureli_pre_commit_config_location( + test_folder_path, True + ) + assert update_result.outcome == VerifyOutcome.UPDATE_FAILED def test_that_update_secureli_pre_commit_config_location_cancels_on_user_response( action: Action, mock_echo: MagicMock, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, ): mock_echo.confirm.return_value = False update_file_location = test_folder_path / ".secureli" / ".pre-commit-config.yaml" @@ -485,7 +507,7 @@ def test_that_update_secureli_pre_commit_config_location_cancels_on_user_respons mock_echo.warning.assert_called_once_with( ".pre-commit-config.yaml migration declined" ) - mock_scanner.pre_commit.migrate_config_file.assert_not_called() + mock_hooks_scanner.pre_commit.migrate_config_file.assert_not_called() assert update_result.outcome == VerifyOutcome.UPDATE_CANCELED @@ -495,7 +517,7 @@ def test_that_initialize_repo_install_flow_warns_about_overwriting_pre_commit_fi mock_echo: MagicMock, mock_updater: MagicMock, ): - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={ "RadLang": 0.75, }, @@ -508,7 +530,7 @@ def test_that_initialize_repo_install_flow_warns_about_overwriting_pre_commit_fi mock_updater.pre_commit.install.return_value = install_result - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) mock_echo.warning.assert_called_once_with( ( @@ -658,7 +680,7 @@ def test_that_post_install_scan_creates_pre_commit_on_new_install( action: Action, mock_updater: MagicMock ): action._run_post_install_scan( - "test/path", SecureliConfig(), LanguageMetadata(version="0.03"), True + "test/path", SecureliConfig(), language.LanguageMetadata(version="0.03"), True ) mock_updater.pre_commit.install.assert_called_once() @@ -668,37 +690,37 @@ def test_that_post_install_scan_ignores_creating_pre_commit_on_existing_install( action: Action, mock_updater: MagicMock ): action._run_post_install_scan( - "test/path", SecureliConfig(), LanguageMetadata(version="0.03"), False + "test/path", SecureliConfig(), language.LanguageMetadata(version="0.03"), False ) mock_updater.pre_commit.install.assert_not_called() def test_that_post_install_scan_scans_repo( - action: Action, mock_scanner: MagicMock, mock_echo: MagicMock + action: Action, mock_hooks_scanner: MagicMock, mock_echo: MagicMock ): action._run_post_install_scan( "test/path", SecureliConfig(), - LanguageMetadata(version="0.03", security_hook_id="secrets-hook"), + language.LanguageMetadata(version="0.03", security_hook_id="secrets-hook"), False, ) - mock_scanner.scan_repo.assert_called_once() + mock_hooks_scanner.scan_repo.assert_called_once() mock_echo.warning.assert_not_called() def test_that_post_install_scan_does_not_scan_repo_when_no_security_hook_id( - action: Action, mock_scanner: MagicMock, mock_echo: MagicMock + action: Action, mock_hooks_scanner: MagicMock, mock_echo: MagicMock ): action._run_post_install_scan( "test/path", SecureliConfig(languages=["RadLang"]), - LanguageMetadata(version="0.03"), + language.LanguageMetadata(version="0.03"), False, ) - mock_scanner.scan_repo.assert_not_called() + mock_hooks_scanner.scan_repo.assert_not_called() mock_echo.warning.assert_called_once_with( "RadLang does not support secrets detection, skipping" ) @@ -707,7 +729,7 @@ def test_that_post_install_scan_does_not_scan_repo_when_no_security_hook_id( def test_that_install_saves_settings( action: Action, mock_language_analyzer: MagicMock, mock_settings: MagicMock ): - mock_language_analyzer.analyze.return_value = AnalyzeResult( + mock_language_analyzer.analyze.return_value = language.AnalyzeResult( language_proportions={"PreviousLang": 1.0}, skipped_files=[], ) diff --git a/tests/actions/test_build_action.py b/tests/actions/test_build_action.py index 22c8b098..1381a996 100644 --- a/tests/actions/test_build_action.py +++ b/tests/actions/test_build_action.py @@ -3,7 +3,7 @@ import pytest from secureli.actions.build import BuildAction -from secureli.models.echo import Color +from secureli.modules.shared.models.echo import Color @pytest.fixture() diff --git a/tests/actions/test_initializer_action.py b/tests/actions/test_initializer_action.py index f46b53de..a26fff90 100644 --- a/tests/actions/test_initializer_action.py +++ b/tests/actions/test_initializer_action.py @@ -5,16 +5,16 @@ from secureli.actions.action import ActionDependencies from secureli.actions.initializer import InitializerAction -from secureli.services.language_config import LanguageNotSupportedError -from secureli.services.logging import LogAction +from secureli.modules.shared.models.language import LanguageNotSupportedError +from secureli.modules.shared.models.logging import LogAction test_folder_path = Path("does-not-matter") @pytest.fixture() -def mock_scanner() -> MagicMock: - mock_scanner = MagicMock() - return mock_scanner +def mock_hooks_scanner() -> MagicMock: + mock_hooks_scanner = MagicMock() + return mock_hooks_scanner @pytest.fixture() @@ -28,7 +28,7 @@ def action_deps( mock_echo: MagicMock, mock_language_analyzer: MagicMock, mock_language_support: MagicMock, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_secureli_config: MagicMock, mock_settings: MagicMock, mock_updater: MagicMock, @@ -37,7 +37,7 @@ def action_deps( mock_echo, mock_language_analyzer, mock_language_support, - mock_scanner, + mock_hooks_scanner, mock_secureli_config, mock_settings, mock_updater, diff --git a/tests/actions/test_scan_action.py b/tests/actions/test_scan_action.py index 3fd1ba0b..8724ada8 100644 --- a/tests/actions/test_scan_action.py +++ b/tests/actions/test_scan_action.py @@ -1,22 +1,17 @@ from pathlib import Path -from secureli.abstractions.pre_commit import RevisionPair -from secureli.actions.action import ActionDependencies, VerifyOutcome +from secureli.modules.shared.abstractions.pre_commit import RevisionPair +from secureli.actions.action import ActionDependencies from secureli.actions.scan import ScanAction -from secureli.models.exit_codes import ExitCode -from secureli.models.publish_results import PublishResultsOption -from secureli.models.result import Result +from secureli.modules.shared.models.echo import Level +from secureli.modules.shared.models.exit_codes import ExitCode +from secureli.modules.shared.models.install import VerifyOutcome +from secureli.modules.shared.models.language import AnalyzeResult +from secureli.modules.shared.models.logging import LogAction +from secureli.modules.shared.models.publish_results import PublishResultsOption +from secureli.modules.shared.models.result import Result +from secureli.modules.shared.models.scan import ScanMode, ScanResult from secureli.repositories.secureli_config import SecureliConfig, VerifyConfigOutcome -from secureli.repositories.settings import ( - PreCommitHook, - PreCommitRepo, - PreCommitSettings, - SecureliFile, - EchoSettings, - EchoLevel, -) -from secureli.services.language_analyzer import AnalyzeResult -from secureli.services.logging import LogAction -from secureli.services.scanner import ScanMode, ScanResult +from secureli.repositories import repo_settings from unittest import mock from unittest.mock import MagicMock, patch from pytest_mock import MockerFixture @@ -30,41 +25,55 @@ @pytest.fixture() -def mock_scanner(mock_pre_commit) -> MagicMock: - mock_scanner = MagicMock() - mock_scanner.scan_repo.return_value = ScanResult(successful=True, failures=[]) - mock_scanner.pre_commit = mock_pre_commit - return mock_scanner +def mock_hooks_scanner(mock_pre_commit) -> MagicMock: + mock_hooks_scanner = MagicMock() + mock_hooks_scanner.scan_repo.return_value = ScanResult(successful=True, failures=[]) + mock_hooks_scanner.pre_commit = mock_pre_commit + return mock_hooks_scanner @pytest.fixture() def mock_pre_commit() -> MagicMock: mock_pre_commit = MagicMock() - mock_pre_commit.get_pre_commit_config.return_value = PreCommitSettings( - repos=[ - PreCommitRepo( - repo="http://example-repo.com/", - rev="master", - hooks=[ - PreCommitHook( - id="hook-id", - arguments=None, - additional_args=None, - ) - ], - ) - ] + mock_pre_commit.get_pre_commit_config.return_value = ( + repo_settings.PreCommitSettings( + repos=[ + repo_settings.PreCommitRepo( + repo="http://example-repo.com/", + rev="master", + hooks=[ + repo_settings.PreCommitHook( + id="hook-id", + arguments=None, + additional_args=None, + ) + ], + ) + ] + ) ) mock_pre_commit.check_for_hook_updates.return_value = {} return mock_pre_commit +@pytest.fixture() +def mock_pii_scanner() -> MagicMock: + mock_pii_scanner = MagicMock() + mock_pii_scanner.scan_repo.return_value = ScanResult(successful=True, failures=[]) + return mock_pii_scanner + + @pytest.fixture() def mock_updater() -> MagicMock: mock_updater = MagicMock() return mock_updater +@pytest.fixture() +def mock_git_repo() -> MagicMock: + return MagicMock() + + @pytest.fixture() def mock_get_time_near_epoch(mocker: MockerFixture) -> MagicMock: return mocker.patch( @@ -79,8 +88,8 @@ def mock_get_time_far_from_epoch(mocker: MockerFixture) -> MagicMock: @pytest.fixture() def mock_default_settings(mock_settings_repository: MagicMock) -> MagicMock: - mock_echo_settings = EchoSettings(level=EchoLevel.info) - mock_settings_file = SecureliFile(echo=mock_echo_settings) + mock_echo_settings = repo_settings.EchoSettings(level=Level.info) + mock_settings_file = repo_settings.SecureliFile(echo=mock_echo_settings) mock_settings_repository.load.return_value = mock_settings_file return mock_settings_repository @@ -101,7 +110,7 @@ def action_deps( mock_echo: MagicMock, mock_language_analyzer: MagicMock, mock_language_support: MagicMock, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_secureli_config: MagicMock, mock_settings_repository: MagicMock, mock_updater: MagicMock, @@ -110,7 +119,7 @@ def action_deps( mock_echo, mock_language_analyzer, mock_language_support, - mock_scanner, + mock_hooks_scanner, mock_secureli_config, mock_settings_repository, mock_updater, @@ -121,24 +130,29 @@ def action_deps( def scan_action( action_deps: ActionDependencies, mock_logging_service: MagicMock, + mock_pii_scanner: MagicMock, + mock_git_repo: MagicMock, ) -> ScanAction: return ScanAction( action_deps=action_deps, echo=action_deps.echo, logging=mock_logging_service, - scanner=action_deps.scanner, + hooks_scanner=action_deps.hooks_scanner, + pii_scanner=mock_pii_scanner, + git_repo=mock_git_repo, ) @pytest.fixture() def mock_post_log(mocker: MockerFixture) -> MagicMock: - return mocker.patch("secureli.actions.scan.post_log") + return mocker.patch("secureli.modules.shared.utilities.post_log") @mock.patch.dict(os.environ, {"API_KEY": "", "API_ENDPOINT": ""}, clear=True) def test_that_scan_repo_errors_if_not_successful( scan_action: ScanAction, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, + mock_pii_scanner: MagicMock, mock_secureli_config: MagicMock, mock_language_analyzer: MagicMock, ): @@ -147,7 +161,10 @@ def test_that_scan_repo_errors_if_not_successful( language_proportions={f"{mock_language}": 1.0}, skipped_files=[], ) - mock_scanner.scan_repo.return_value = ScanResult( + mock_pii_scanner.scan_repo.return_value = ScanResult( + successful=False, output="So much PII", failures=[] + ) + mock_hooks_scanner.scan_repo.return_value = ScanResult( successful=False, output="Bad Error", failures=[] ) mock_secureli_config.load.return_value = SecureliConfig( @@ -165,7 +182,7 @@ def test_that_scan_repo_scans_if_installed( scan_action: ScanAction, mock_secureli_config: MagicMock, mock_language_support: MagicMock, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_language_analyzer: MagicMock, ): mock_language_analyzer.analyze.return_value = AnalyzeResult( @@ -177,9 +194,45 @@ def test_that_scan_repo_scans_if_installed( ) mock_language_support.version_for_language.return_value = "abc123" - scan_action.scan_repo(test_folder_path, ScanMode.STAGED_ONLY, False) + scan_action.scan_repo( + test_folder_path, ScanMode.STAGED_ONLY, False, None, "detect-secrets" + ) - mock_scanner.scan_repo.assert_called_once() + mock_hooks_scanner.scan_repo.assert_called_once() + + +@mock.patch.dict(os.environ, {"API_KEY": "", "API_ENDPOINT": ""}, clear=True) +def test_that_scan_repo_conducts_all_scans_and_merges_results( + scan_action: ScanAction, + mock_secureli_config: MagicMock, + mock_language_support: MagicMock, + mock_hooks_scanner: MagicMock, + mock_pii_scanner: MagicMock, + mock_language_analyzer: MagicMock, + mock_echo: MagicMock, +): + mock_language_analyzer.analyze.return_value = AnalyzeResult( + language_proportions={"RadLang": 1.0}, + skipped_files=[], + ) + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed="abc123" + ) + mock_language_support.version_for_language.return_value = "abc123" + mock_failure_1 = "Hooks scan failure" + mock_failure_2 = "PII scan failure" + mock_hooks_scanner.scan_repo.return_value = ScanResult( + successful=False, failures=[], output=mock_failure_1 + ) + mock_pii_scanner.scan_repo.return_value = ScanResult( + successful=False, failures=[], output=mock_failure_2 + ) + + with pytest.raises(SystemExit): + scan_action.scan_repo(test_folder_path, ScanMode.STAGED_ONLY, False) + mock_hooks_scanner.scan_repo.assert_called_once() + mock_pii_scanner.scan_repo.assert_called_once() + mock_echo.print.assert_called_once_with(f"\n{mock_failure_1}\n{mock_failure_2}") @mock.patch.dict(os.environ, {"API_KEY": "", "API_ENDPOINT": ""}, clear=True) @@ -187,7 +240,8 @@ def test_that_scan_repo_continue_scan_if_upgrade_canceled( scan_action: ScanAction, mock_secureli_config: MagicMock, mock_language_support: MagicMock, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, + mock_pii_scanner: MagicMock, mock_echo: MagicMock, mock_language_analyzer: MagicMock, ): @@ -203,13 +257,15 @@ def test_that_scan_repo_continue_scan_if_upgrade_canceled( scan_action.scan_repo(test_folder_path, ScanMode.STAGED_ONLY, False) - mock_scanner.scan_repo.assert_called_once() + mock_hooks_scanner.scan_repo.assert_called_once() + mock_pii_scanner.scan_repo.assert_called_once() @mock.patch.dict(os.environ, {"API_KEY": "", "API_ENDPOINT": ""}, clear=True) def test_that_scan_repo_does_not_scan_if_not_installed( scan_action: ScanAction, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, + mock_pii_scanner: MagicMock, mock_secureli_config: MagicMock, mock_echo: MagicMock, mock_language_analyzer: MagicMock, @@ -218,48 +274,49 @@ def test_that_scan_repo_does_not_scan_if_not_installed( mock_secureli_config.load.return_value = SecureliConfig() mock_secureli_config.verify.return_value = VerifyConfigOutcome.UP_TO_DATE mock_echo.confirm.return_value = False - # mock_language_analyzer._detect_languages.side_effect = Exception("Error") + scan_action.scan_repo(test_folder_path, ScanMode.STAGED_ONLY, False) - mock_scanner.scan_repo.assert_not_called() + mock_hooks_scanner.scan_repo.assert_not_called() + mock_pii_scanner.scan_repo.assert_not_called() def test_that_scan_checks_for_updates( scan_action: ScanAction, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_secureli_config: MagicMock, mock_pass_install_verification: MagicMock, ): scan_action.scan_repo(test_folder_path, ScanMode.STAGED_ONLY, always_yes=True) - mock_scanner.pre_commit.check_for_hook_updates.assert_called_once() + mock_hooks_scanner.pre_commit.check_for_hook_updates.assert_called_once() def test_that_scan_only_checks_for_updates_periodically( scan_action: ScanAction, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_get_time_near_epoch: MagicMock, mock_secureli_config: MagicMock, ): mock_secureli_config.load.return_value = SecureliConfig() scan_action.scan_repo(test_folder_path, ScanMode.STAGED_ONLY, always_yes=True) - mock_scanner.pre_commit.check_for_hook_updates.assert_not_called() + mock_hooks_scanner.pre_commit.check_for_hook_updates.assert_not_called() def test_that_scan_update_check_uses_pre_commit_config( scan_action: ScanAction, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_secureli_config: MagicMock, ): mock_secureli_config.load.return_value = SecureliConfig() scan_action.scan_repo(test_folder_path, ScanMode.STAGED_ONLY, always_yes=True) - mock_scanner.pre_commit.get_pre_commit_config.assert_called_once() + mock_hooks_scanner.pre_commit.get_pre_commit_config.assert_called_once() # Test that _check_secureli_hook_updates returns UP_TO_DATE if no hooks need updating def test_scan_update_check_return_value_when_up_to_date( scan_action: ScanAction, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_secureli_config: MagicMock, ): mock_secureli_config.load.return_value = SecureliConfig() @@ -270,11 +327,11 @@ def test_scan_update_check_return_value_when_up_to_date( # Test that _check_secureli_hook_updates returns UPDATE_CANCELED if hooks need updating def test_scan_update_check_return_value_when_not_up_to_date( scan_action: ScanAction, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_secureli_config: MagicMock, ): mock_secureli_config.load.return_value = SecureliConfig() - mock_scanner.pre_commit.check_for_hook_updates.return_value = { + mock_hooks_scanner.pre_commit.check_for_hook_updates.return_value = { "http://example-repo.com/": RevisionPair(oldRev="old-rev", newRev="new-rev") } result = scan_action._check_secureli_hook_updates(test_folder_path) @@ -284,7 +341,7 @@ def test_scan_update_check_return_value_when_not_up_to_date( # Validate that scan_repo persists changes to the .secureli.yaml file after checking for hook updates def test_that_scan_update_check_updates_last_check_time( scan_action: ScanAction, - mock_scanner: MagicMock, + mock_hooks_scanner: MagicMock, mock_get_time_far_from_epoch: MagicMock, mock_secureli_config: MagicMock, mock_pass_install_verification: MagicMock, @@ -339,3 +396,116 @@ def test_publish_results_on_fail_and_action_not_successful( mock_post_log.assert_called_once_with("log_str", Settings()) scan_action.logging.failure.assert_called_once_with(LogAction.publish, "Failure") + + +def test_verify_install_is_called_with_commted_files( + scan_action: ScanAction, + mock_git_repo: MagicMock, + mock_secureli_config: MagicMock, + mock_language_analyzer: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed=1 + ) + + mock_files = ["file1.py", "file2.py"] + + mock_git_repo.get_commit_diff.return_value = mock_files + scan_action.scan_repo( + folder_path=Path(""), + scan_mode=ScanMode.STAGED_ONLY, + always_yes=True, + publish_results_condition=PublishResultsOption.NEVER, + specific_test=None, + files=None, + ) + + mock_language_analyzer.analyze.assert_called_once_with( + Path("."), [Path(file) for file in mock_files] + ) + + +def test_verify_install_is_called_with_user_specified_files( + scan_action: ScanAction, + mock_git_repo: MagicMock, + mock_secureli_config: MagicMock, + mock_language_analyzer: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed=1 + ) + + mock_files = ["file1.py", "file2.py"] + + mock_git_repo.get_commit_diff.return_value = None + scan_action.scan_repo( + folder_path=Path(""), + scan_mode=ScanMode.STAGED_ONLY, + always_yes=True, + publish_results_condition=PublishResultsOption.NEVER, + specific_test=None, + files=mock_files, + ) + + mock_language_analyzer.analyze.assert_called_once_with( + Path("."), [Path(file) for file in mock_files] + ) + + +def test_verify_install_is_called_with_no_specified_files( + scan_action: ScanAction, + mock_git_repo: MagicMock, + mock_secureli_config: MagicMock, + mock_language_analyzer: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed=1 + ) + + mock_git_repo.get_commit_diff.return_value = None + scan_action.scan_repo( + folder_path=Path(""), + scan_mode=ScanMode.STAGED_ONLY, + always_yes=True, + publish_results_condition=PublishResultsOption.NEVER, + specific_test=None, + files=None, + ) + + mock_language_analyzer.analyze.assert_called_once_with(Path("."), None) + + +def test_get_commited_files_returns_commit_diff( + scan_action: ScanAction, + mock_git_repo: MagicMock, + mock_secureli_config: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed=1 + ) + mock_files = [Path("file1.py"), Path("file2.py")] + mock_git_repo.get_commit_diff.return_value = mock_files + result = scan_action.get_commited_files(scan_mode=ScanMode.STAGED_ONLY) + assert result == mock_files + + +def test_get_commited_files_returns_none_when_not_installed( + scan_action: ScanAction, + mock_secureli_config: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=[], version_installed=None + ) + result = scan_action.get_commited_files(scan_mode=ScanMode.STAGED_ONLY) + assert result is None + + +def test_get_commited_files_returns_when_scan_mode_is_not_staged_only( + scan_action: ScanAction, + mock_secureli_config: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed=1 + ) + result = scan_action.get_commited_files(scan_mode=ScanMode.ALL_FILES) + assert result is None diff --git a/tests/actions/test_update_action.py b/tests/actions/test_update_action.py index fbce1cf3..08c51479 100644 --- a/tests/actions/test_update_action.py +++ b/tests/actions/test_update_action.py @@ -4,7 +4,7 @@ from secureli.actions.action import ActionDependencies from secureli.actions.update import UpdateAction -from secureli.services.updater import UpdateResult +from secureli.modules.core.core_services.updater import UpdateResult test_folder_path = Path("does-not-matter") diff --git a/tests/application/test_main.py b/tests/application/test_main.py index 2c03d775..a16d7c0e 100644 --- a/tests/application/test_main.py +++ b/tests/application/test_main.py @@ -4,13 +4,13 @@ import pytest from pytest_mock import MockerFixture -from secureli.actions.action import VerifyOutcome, VerifyResult import secureli.container import secureli.main -from secureli.models.publish_results import PublishResultsOption -from secureli.services.scanner import ScanMode -from secureli.utilities.secureli_meta import secureli_version +from secureli.modules.shared.models.install import VerifyOutcome, VerifyResult +from secureli.modules.shared.models.publish_results import PublishResultsOption +from secureli.modules.shared.models.scan import ScanMode +from secureli.modules.shared.utilities import secureli_version @pytest.fixture() diff --git a/tests/application/test_settings.py b/tests/application/test_settings.py index 9d2ba222..7efa067f 100644 --- a/tests/application/test_settings.py +++ b/tests/application/test_settings.py @@ -2,10 +2,7 @@ from pytest_mock import MockerFixture -from secureli.settings import ( - Settings, - secureli_yaml_settings, -) +from secureli import settings def test_that_secureli_yaml_settings_guards_against_missing_yaml_file( @@ -16,7 +13,7 @@ def test_that_secureli_yaml_settings_guards_against_missing_yaml_file( path_class = mocker.patch("secureli.settings.Path") path_class.return_value = path_instance - assert not secureli_yaml_settings(Settings()) + assert not settings.secureli_yaml_settings(settings.Settings()) def test_that_secureli_yaml_settings_processes_present_yaml_file( @@ -34,6 +31,6 @@ def test_that_secureli_yaml_settings_processes_present_yaml_file( ) mocker.patch("builtins.open", mock_open) - result = secureli_yaml_settings(Settings()) + result = settings.secureli_yaml_settings(settings.Settings()) assert "language_support" in result assert result["language_support"]["command_timeout_seconds"] == 12345 diff --git a/tests/conftest.py b/tests/conftest.py index 6ba83b09..d8608aa6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ import pytest from pytest_mock import MockerFixture -from secureli.abstractions.pre_commit import ExecuteResult +from secureli.modules.shared.abstractions.pre_commit import ExecuteResult # Register generic mocks you'd like available for every test. diff --git a/tests/end-to-end/testinstallnewhooks.bats b/tests/end-to-end/testinstallnewhooks.bats new file mode 100644 index 00000000..12bf185b --- /dev/null +++ b/tests/end-to-end/testinstallnewhooks.bats @@ -0,0 +1,22 @@ +# Test to ensure pre-exisiting hooks within the .pre-commit-config.yaml files +# are persisted when installing secureli + +MOCK_REPO='tests/test-data/mock-repo' + +setup() { + load "${BATS_LIBS_ROOT}/bats-support/load" + load "${BATS_LIBS_ROOT}/bats-assert/load" + mkdir -p $MOCK_REPO + echo 'print("hello world!")' > $MOCK_REPO/hw.py + run git init $MOCK_REPO +} + +@test "can preserve pre-existing hooks" { + run python secureli/main.py init -y --directory $MOCK_REPO + run grep 'https://github.com/psf/black' $MOCK_REPO/.secureli/.pre-commit-config.yaml + assert_output --partial 'https://github.com/psf/black' +} + +teardown() { + rm -rf $MOCK_REPO +} diff --git a/tests/end-to-end/testpreservehooks.bats b/tests/end-to-end/testpreservehooks.bats index 13653bd8..30d653e4 100644 --- a/tests/end-to-end/testpreservehooks.bats +++ b/tests/end-to-end/testpreservehooks.bats @@ -7,10 +7,9 @@ setup() { load "${BATS_LIBS_ROOT}/bats-support/load" load "${BATS_LIBS_ROOT}/bats-assert/load" mkdir -p $MOCK_REPO - echo '# Existing YAML Contents should persist' > $MOCK_REPO/.pre-commit-config.yaml echo 'repos:' >> $MOCK_REPO/.pre-commit-config.yaml echo '- repo: https://github.com/hhatto/autopep8' >> $MOCK_REPO/.pre-commit-config.yaml - echo ' rev: v2.0.4' >> $MOCK_REPO/.pre-commit-config.yaml + echo ' rev: v2.1.0' >> $MOCK_REPO/.pre-commit-config.yaml echo ' hooks:' >> $MOCK_REPO/.pre-commit-config.yaml echo ' - id: autopep8' >> $MOCK_REPO/.pre-commit-config.yaml echo 'fail_fast: false' >> $MOCK_REPO/.pre-commit-config.yaml diff --git a/tests/services/.pre-commit-config.yaml b/tests/modules/.pre-commit-config.yaml similarity index 100% rename from tests/services/.pre-commit-config.yaml rename to tests/modules/.pre-commit-config.yaml diff --git a/tests/services/__init__.py b/tests/modules/__init__.py similarity index 100% rename from tests/services/__init__.py rename to tests/modules/__init__.py diff --git a/tests/services/test_scanner_service.py b/tests/modules/core/test_scanner_service.py similarity index 85% rename from tests/services/test_scanner_service.py rename to tests/modules/core/test_scanner_service.py index cfd20e30..820bb9f1 100644 --- a/tests/services/test_scanner_service.py +++ b/tests/modules/core/test_scanner_service.py @@ -2,13 +2,13 @@ from pathlib import Path import pytest -from secureli.abstractions.pre_commit import ExecuteResult -from secureli.repositories.settings import ( - PreCommitHook, - PreCommitRepo, - PreCommitSettings, +from secureli.modules.shared.abstractions.pre_commit import ExecuteResult +from secureli.modules.shared.models.scan import ScanMode +from secureli.repositories import repo_settings +from secureli.modules.core.core_services.scanner import ( + HooksScannerService, + OutputParseErrors, ) -from secureli.services.scanner import ScannerService, ScanMode, OutputParseErrors from pytest_mock import MockerFixture test_folder_path = Path(".") @@ -114,12 +114,12 @@ def mock_config_no_black(mocker: MockerFixture) -> MagicMock: @pytest.fixture() -def scanner_service(mock_pre_commit: MagicMock) -> ScannerService: - return ScannerService(mock_pre_commit) +def scanner_service(mock_pre_commit: MagicMock) -> HooksScannerService: + return HooksScannerService(mock_pre_commit) def test_that_scanner_service_scans_repositories_with_pre_commit( - scanner_service: ScannerService, + scanner_service: HooksScannerService, mock_pre_commit: MagicMock, ): scan_result = scanner_service.scan_repo(test_folder_path, ScanMode.ALL_FILES) @@ -129,7 +129,7 @@ def test_that_scanner_service_scans_repositories_with_pre_commit( def test_that_scanner_service_parses_failures( - scanner_service: ScannerService, + scanner_service: HooksScannerService, mock_pre_commit: MagicMock, mock_scan_output_single_failure: MagicMock, mock_config_all_repos: MagicMock, @@ -143,7 +143,7 @@ def test_that_scanner_service_parses_failures( def test_that_scanner_service_parses_multiple_failures( - scanner_service: ScannerService, + scanner_service: HooksScannerService, mock_pre_commit: MagicMock, mock_scan_output_double_failure: MagicMock, mock_config_all_repos: MagicMock, @@ -157,7 +157,7 @@ def test_that_scanner_service_parses_multiple_failures( def test_that_scanner_service_parses_when_no_failures( - scanner_service: ScannerService, + scanner_service: HooksScannerService, mock_pre_commit: MagicMock, mock_scan_output_no_failure: MagicMock, mock_config_all_repos: MagicMock, @@ -171,7 +171,7 @@ def test_that_scanner_service_parses_when_no_failures( def test_that_scanner_service_handles_error_in_missing_repo( - scanner_service: ScannerService, + scanner_service: HooksScannerService, mock_pre_commit: MagicMock, mock_scan_output_double_failure: MagicMock, mock_config_no_black: MagicMock, @@ -184,18 +184,20 @@ def test_that_scanner_service_handles_error_in_missing_repo( assert scan_result.failures[1].repo == OutputParseErrors.REPO_NOT_FOUND -def test_that_find_repo_from_id_finds_matching_hooks(scanner_service: ScannerService): +def test_that_find_repo_from_id_finds_matching_hooks( + scanner_service: HooksScannerService, +): mock_hook_id = "find_secrets" expected_repo = "mock_repo" result = scanner_service._find_repo_from_id( mock_hook_id, - PreCommitSettings( + repo_settings.PreCommitSettings( repos=[ - PreCommitRepo( + repo_settings.PreCommitRepo( repo=expected_repo, rev="", url="test-url", - hooks=[PreCommitHook(id=mock_hook_id)], + hooks=[repo_settings.PreCommitHook(id=mock_hook_id)], suppressed_hook_ids=[], ) ], @@ -207,17 +209,17 @@ def test_that_find_repo_from_id_finds_matching_hooks(scanner_service: ScannerSer def test_that_find_repo_from_id_does_not_have_matching_hook_id( - scanner_service: ScannerService, + scanner_service: HooksScannerService, ): result = scanner_service._find_repo_from_id( "test-hook-id", - PreCommitSettings( + repo_settings.PreCommitSettings( repos=[ - PreCommitRepo( + repo_settings.PreCommitRepo( repo="mock-repo", rev="", url="test-url", - hooks=[PreCommitHook(id="other_hook_id")], + hooks=[repo_settings.PreCommitHook(id="other_hook_id")], suppressed_hook_ids=[], ) ], diff --git a/tests/services/test_updater_service.py b/tests/modules/core/test_updater_service.py similarity index 95% rename from tests/services/test_updater_service.py rename to tests/modules/core/test_updater_service.py index e4ff16bf..5fd818d8 100644 --- a/tests/services/test_updater_service.py +++ b/tests/modules/core/test_updater_service.py @@ -2,8 +2,8 @@ from pathlib import Path import pytest -from secureli.abstractions.pre_commit import ExecuteResult -from secureli.services.updater import UpdaterService +from secureli.modules.shared.abstractions.pre_commit import ExecuteResult +from secureli.modules.core.core_services.updater import UpdaterService test_folder_path = Path("does-not-matter") diff --git a/tests/services/test_git_ignore.py b/tests/modules/language_analyzer/test_git_ignore.py similarity index 75% rename from tests/services/test_git_ignore.py rename to tests/modules/language_analyzer/test_git_ignore.py index b8653e08..63c31474 100644 --- a/tests/services/test_git_ignore.py +++ b/tests/modules/language_analyzer/test_git_ignore.py @@ -5,7 +5,7 @@ import pytest from pytest_mock import MockerFixture -from secureli.services.git_ignore import GitIgnoreService, BadIgnoreBlockError +from secureli.modules.language_analyzer import git_ignore @pytest.fixture() @@ -44,46 +44,48 @@ def mock_open_with_gitignore_broken_secureli_config(mocker: MockerFixture) -> Ma @pytest.fixture() def mock_path(mocker: MockerFixture) -> MagicMock: - mock_path_class = mocker.patch("secureli.services.git_ignore.Path") + mock_path_class = mocker.patch("secureli.modules.language_analyzer.git_ignore.Path") mock_path_instance = MagicMock() mock_path_class.return_value = mock_path_instance return mock_path_instance @pytest.fixture -def git_ignore(mock_path: MagicMock) -> GitIgnoreService: - git_ignore = GitIgnoreService() - git_ignore.git_ignore_path = mock_path - return git_ignore +def git_ignore_fixture(mock_path: MagicMock) -> git_ignore.GitIgnoreService: + git_ignore_fixture = git_ignore.GitIgnoreService() + git_ignore_fixture.git_ignore_path = mock_path + return git_ignore_fixture def test_that_git_ignore_creates_file_if_missing( - git_ignore: GitIgnoreService, mock_path: MagicMock, mock_open: MagicMock + git_ignore_fixture: git_ignore.GitIgnoreService, + mock_path: MagicMock, + mock_open: MagicMock, ): mock_path.exists.return_value = False with um.patch.object(Path, "exists") as mock_exists: mock_exists.return_value = False - git_ignore.ignore_secureli_files() + git_ignore_fixture.ignore_secureli_files() mock_open.return_value.write.assert_called_once() args, _ = mock_open.return_value.write.call_args_list[0] assert args[0].find("# existing contents") == -1 assert args[0].find(".secureli") != -1 - assert args[0].find(git_ignore.header) != -1 - assert args[0].find(git_ignore.footer) != -1 + assert args[0].find(git_ignore_fixture.header) != -1 + assert args[0].find(git_ignore_fixture.footer) != -1 def test_that_git_ignore_appends_to_existing_file_if_block_is_missing( - git_ignore: GitIgnoreService, + git_ignore_fixture: git_ignore.GitIgnoreService, mock_path: MagicMock, mock_open_with_gitignore: MagicMock, ): mock_path.exists.return_value = True - git_ignore.ignore_secureli_files() + git_ignore_fixture.ignore_secureli_files() mock_open_with_gitignore.return_value.write.assert_called_once() @@ -93,13 +95,13 @@ def test_that_git_ignore_appends_to_existing_file_if_block_is_missing( def test_that_git_ignore_updates_existing_file_if_block_is_present( - git_ignore: GitIgnoreService, + git_ignore_fixture: git_ignore.GitIgnoreService, mock_path: MagicMock, mock_open_with_gitignore_existing_secureli_config: MagicMock, ): mock_path.exists.return_value = True - git_ignore.ignore_secureli_files() + git_ignore_fixture.ignore_secureli_files() mock_open_with_gitignore_existing_secureli_config.return_value.write.assert_called_once() @@ -115,24 +117,24 @@ def test_that_git_ignore_updates_existing_file_if_block_is_present( def test_that_git_ignore_is_mad_if_header_is_found_without_footer( - git_ignore: GitIgnoreService, + git_ignore_fixture: git_ignore.GitIgnoreService, mock_path: MagicMock, mock_open_with_gitignore_broken_secureli_config: MagicMock, ): mock_path.exists.return_value = True - with pytest.raises(BadIgnoreBlockError): - git_ignore.ignore_secureli_files() + with pytest.raises(git_ignore.BadIgnoreBlockError): + git_ignore_fixture.ignore_secureli_files() def test_that_ignore_ignore_finds_and_reads_file( - git_ignore: GitIgnoreService, + git_ignore_fixture: git_ignore.GitIgnoreService, mock_path: MagicMock, mock_open_with_ignored_patterns: MagicMock, ): mock_path.exists.return_value = True - ignored_patterns = git_ignore.ignored_file_patterns() + ignored_patterns = git_ignore_fixture.ignored_file_patterns() assert ignored_patterns == [ "^(?:.+/)?[^/]*\\.py(?:(?P/).*)?$", @@ -141,12 +143,12 @@ def test_that_ignore_ignore_finds_and_reads_file( def test_that_ignore_ignore_does_not_find_file_and_returns_empty( - git_ignore: GitIgnoreService, + git_ignore_fixture: git_ignore.GitIgnoreService, mock_path: MagicMock, mock_open_with_ignored_patterns: MagicMock, ): mock_path.exists.return_value = False - ignored_patterns = git_ignore.ignored_file_patterns() + ignored_patterns = git_ignore_fixture.ignored_file_patterns() assert ignored_patterns == [] diff --git a/tests/services/test_language_analyzer.py b/tests/modules/language_analyzer/test_language_analyzer.py similarity index 77% rename from tests/services/test_language_analyzer.py rename to tests/modules/language_analyzer/test_language_analyzer.py index 4356dafe..e5ea9680 100644 --- a/tests/services/test_language_analyzer.py +++ b/tests/modules/language_analyzer/test_language_analyzer.py @@ -3,7 +3,7 @@ import pytest -from secureli.services.language_analyzer import LanguageAnalyzerService +from secureli.modules.language_analyzer import language_analyzer @pytest.fixture() @@ -48,8 +48,8 @@ def mock_lexer_guesser_python() -> MagicMock: def language_analyzer_bad_lang( mock_repo_files: MagicMock, mock_lexer_guesser_bad_lang: MagicMock, -) -> LanguageAnalyzerService: - return LanguageAnalyzerService( +) -> language_analyzer.LanguageAnalyzerService: + return language_analyzer.LanguageAnalyzerService( repo_files=mock_repo_files, lexer_guesser=mock_lexer_guesser_bad_lang, ) @@ -59,8 +59,8 @@ def language_analyzer_bad_lang( def language_analyzer_python( mock_repo_files: MagicMock, mock_lexer_guesser_python: MagicMock, -) -> LanguageAnalyzerService: - return LanguageAnalyzerService( +) -> language_analyzer.LanguageAnalyzerService: + return language_analyzer.LanguageAnalyzerService( repo_files=mock_repo_files, lexer_guesser=mock_lexer_guesser_python, ) @@ -70,8 +70,8 @@ def language_analyzer_python( def language_analyzer_with_warnings( mock_repo_files_value_error: MagicMock, mock_lexer_guesser_python: MagicMock, -) -> LanguageAnalyzerService: - return LanguageAnalyzerService( +) -> language_analyzer.LanguageAnalyzerService: + return language_analyzer.LanguageAnalyzerService( repo_files=mock_repo_files_value_error, lexer_guesser=mock_lexer_guesser_python, ) @@ -84,26 +84,30 @@ def folder_path() -> MagicMock: def test_that_language_analyzer_removes_unsupported_languages( - language_analyzer_bad_lang: LanguageAnalyzerService, folder_path: MagicMock + language_analyzer_bad_lang: language_analyzer.LanguageAnalyzerService, + folder_path: MagicMock, ): - percentages_per_language = language_analyzer_bad_lang.analyze(folder_path) + percentages_per_language = language_analyzer_bad_lang.analyze( + folder_path, files=None + ) assert "BadLang" not in percentages_per_language def test_that_language_analyzer_includes_python( - language_analyzer_python: LanguageAnalyzerService, folder_path: MagicMock + language_analyzer_python: language_analyzer.LanguageAnalyzerService, + folder_path: MagicMock, ): - analyze_result = language_analyzer_python.analyze(folder_path) + analyze_result = language_analyzer_python.analyze(folder_path, files=None) assert "Python" in analyze_result.language_proportions assert analyze_result.language_proportions["Python"] == 1.0 def test_that_language_analyzer_displays_warnings( - language_analyzer_with_warnings: LanguageAnalyzerService, + language_analyzer_with_warnings: language_analyzer.LanguageAnalyzerService, folder_path: MagicMock, ): - analyze_result = language_analyzer_with_warnings.analyze(folder_path) + analyze_result = language_analyzer_with_warnings.analyze(folder_path, files=None) assert len(analyze_result.skipped_files) == 3 diff --git a/tests/services/test_language_config.py b/tests/modules/language_analyzer/test_language_config.py similarity index 79% rename from tests/services/test_language_config.py rename to tests/modules/language_analyzer/test_language_config.py index 4464af75..05637777 100644 --- a/tests/services/test_language_config.py +++ b/tests/modules/language_analyzer/test_language_config.py @@ -1,12 +1,8 @@ from unittest.mock import MagicMock import pytest - -from secureli.services.language_config import ( - LanguageConfigService, - LanguageNotSupportedError, - LoadLinterConfigsResult, -) +from secureli.modules.language_analyzer import language_config +from secureli.modules.shared.models import language @pytest.fixture() @@ -19,8 +15,8 @@ def mock_data_loader() -> MagicMock: @pytest.fixture() def language_config_service( mock_data_loader: MagicMock, -) -> LanguageConfigService: - return LanguageConfigService( +) -> language_config.LanguageConfigService: + return language_config.LanguageConfigService( command_timeout_seconds=300, data_loader=mock_data_loader, ignored_file_patterns=[], @@ -28,25 +24,25 @@ def language_config_service( def test_that_language_config_service_treats_missing_templates_as_unsupported_language( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, mock_data_loader: MagicMock, ): mock_data_loader.side_effect = ValueError - with pytest.raises(LanguageNotSupportedError): + with pytest.raises(language.LanguageNotSupportedError): language_config_service.get_language_config("BadLang", include_linter=True) def test_that_language_config_service_treats_missing_templates_as_unsupported_language_when_checking_versions( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, mock_data_loader: MagicMock, ): mock_data_loader.side_effect = ValueError - with pytest.raises(LanguageNotSupportedError): + with pytest.raises(language.LanguageNotSupportedError): language_config_service.get_language_config("BadLang", True) def test_that_version_identifiers_are_calculated_for_known_languages( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, ): version = language_config_service.get_language_config( "Python", include_linter=True @@ -56,7 +52,7 @@ def test_that_version_identifiers_are_calculated_for_known_languages( def test_that_language_config_service_templates_are_loaded_with_global_exclude_if_provided_multiple_patterns( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, ): language_config_service.ignored_file_patterns = [ "mock_pattern1", @@ -68,7 +64,7 @@ def test_that_language_config_service_templates_are_loaded_with_global_exclude_i def test_that_language_config_service_templates_are_loaded_without_exclude( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, ): language_config_service.ignored_file_patterns = [] result = language_config_service.get_language_config("Python", include_linter=True) @@ -77,18 +73,18 @@ def test_that_language_config_service_templates_are_loaded_without_exclude( def test_that_language_config_service_templates_are_loaded_without_linter_config_if_include_linter_is_false( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, ): language_config_service.ignored_file_patterns = [] result = language_config_service.get_language_config("Python", include_linter=False) - assert result.linter_config == LoadLinterConfigsResult( + assert result.linter_config == language.LoadLinterConfigsResult( successful=True, linter_data=[] ) def test_that_language_config_service_does_nothing_when_pre_commit_settings_is_empty( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, mock_data_loader: MagicMock, ): def mock_loader_side_effect(resource): @@ -111,7 +107,7 @@ def mock_loader_side_effect(resource): # ### _load_language_config_files #### def test_that_language_config_service_langauge_config_gets_loaded( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, ): result = language_config_service._load_linter_config_file("JavaScript") @@ -119,7 +115,7 @@ def test_that_language_config_service_langauge_config_gets_loaded( def test_that_language_config_service_language_config_does_not_get_loaded( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, ): result = language_config_service._load_linter_config_file("RadLang") @@ -127,7 +123,7 @@ def test_that_language_config_service_language_config_does_not_get_loaded( def test_that_language_config_service_templates_are_loaded_with_global_exclude_if_provided( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, ): language_config_service.ignored_file_patterns = ["mock_pattern"] result = language_config_service.get_language_config("Python", include_linter=True) @@ -136,7 +132,7 @@ def test_that_language_config_service_templates_are_loaded_with_global_exclude_i def test_that_calculate_combined_configuration_adds_lint_config( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, mock_data_loader: MagicMock, ): mock_scanner_config = "repos: [{ repo: 'scanner-pre-commit'}]" @@ -162,7 +158,7 @@ def data_loader_side_effect(*args, **kwargs): def test_that_calculate_combined_configuration_ignores_lint_config( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, mock_data_loader: MagicMock, ): mock_data_loader.return_value = "repos: [{ repo: 'scanner-pre-commit'}]" @@ -175,7 +171,7 @@ def test_that_calculate_combined_configuration_ignores_lint_config( def test_that_calculate_combined_configuration_returns_valid_config_if_config_file_is_empty( - language_config_service: LanguageConfigService, + language_config_service: language_config.LanguageConfigService, mock_data_loader: MagicMock, ): mock_data_loader.return_value = "" diff --git a/tests/services/test_language_support.py b/tests/modules/language_analyzer/test_language_support.py similarity index 81% rename from tests/services/test_language_support.py rename to tests/modules/language_analyzer/test_language_support.py index da1f770d..37733dfa 100644 --- a/tests/services/test_language_support.py +++ b/tests/modules/language_analyzer/test_language_support.py @@ -1,25 +1,17 @@ -from unittest.mock import MagicMock, patch -from pathlib import Path - -import yaml import pytest +import yaml + from _pytest.python_api import raises from pytest_mock import MockerFixture +from unittest.mock import MagicMock, patch +from pathlib import Path + +from secureli.modules.shared.abstractions import pre_commit +from secureli.modules.language_analyzer import language_support, language_config +from secureli.modules.shared.models.config import LinterConfig, LinterConfigData +from secureli.modules.shared.models import language -from secureli.abstractions.pre_commit import ( - InstallResult, -) -from secureli.services.language_support import ( - LanguageSupportService, - LinterConfig, - LinterConfigData, - LinterConfigWriteResult, -) -from secureli.services.language_config import ( - LanguageConfigService, - LanguagePreCommitResult, - LoadLinterConfigsResult, -) +test_folder_path = Path("does-not-matter") test_folder_path = Path("does-not-matter") @@ -51,7 +43,7 @@ def mock_hashlib(mocker: MockerFixture) -> MagicMock: mock_md5 = MagicMock() mock_hashlib.md5.return_value = mock_md5 mock_md5.hexdigest.return_value = "mock-hash-code" - mocker.patch("secureli.utilities.hash.hashlib", mock_hashlib) + mocker.patch("secureli.modules.shared.utilities.hashlib", mock_hashlib) return mock_hashlib @@ -61,14 +53,14 @@ def mock_hashlib_no_match(mocker: MockerFixture) -> MagicMock: mock_md5 = MagicMock() mock_hashlib.md5.return_value = mock_md5 mock_md5.hexdigest.side_effect = ["first-hash-code", "second-hash-code"] - mocker.patch("secureli.utilities.hash.hashlib", mock_hashlib) + mocker.patch("secureli.modules.shared.utilities.hashlib", mock_hashlib) return mock_hashlib @pytest.fixture() def mock_pre_commit_hook() -> MagicMock: mock_pre_commit_hook = MagicMock() - mock_pre_commit_hook.install.return_value = InstallResult( + mock_pre_commit_hook.install.return_value = pre_commit.InstallResult( successful=True, ) @@ -95,7 +87,7 @@ def mock_echo() -> MagicMock: @pytest.fixture() -def mock_language_config_service() -> LanguageConfigService: +def mock_language_config_service() -> language_config.LanguageConfigService: mock_language_config_service = MagicMock() return mock_language_config_service @@ -108,8 +100,8 @@ def language_support_service( mock_language_config_service: MagicMock, mock_data_loader: MagicMock, mock_echo: MagicMock, -) -> LanguageSupportService: - return LanguageSupportService( +) -> language_support.LanguageSupportService: + return language_support.LanguageSupportService( pre_commit_hook=mock_pre_commit_hook, git_ignore=mock_git_ignore, language_config=mock_language_config_service, @@ -119,7 +111,7 @@ def language_support_service( def test_that_language_support_identifies_a_security_hook_we_can_use_during_init( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_data_loader: MagicMock, mock_language_config_service: MagicMock, ): @@ -129,10 +121,12 @@ def mock_loader_side_effect(resource): - baddie-finder-hook """ - mock_language_config_service.get_language_config.return_value = LanguagePreCommitResult( + mock_language_config_service.get_language_config.return_value = language.LanguagePreCommitResult( language="Python", version="abc123", - linter_config=LoadLinterConfigsResult(successful=False, linter_data=list()), + linter_config=language.LoadLinterConfigsResult( + successful=False, linter_data=list() + ), config_data=""" repos: - repo: http://sample-repo.com/baddie-finder @@ -149,7 +143,7 @@ def mock_loader_side_effect(resource): def test_that_language_support_does_not_identify_a_security_hook_if_config_does_not_use_repo_even_if_hook_id_matches( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_data_loader: MagicMock, mock_language_config_service: MagicMock, ): @@ -159,10 +153,12 @@ def mock_loader_side_effect(resource): - baddie-finder-hook """ - mock_language_config_service.get_language_config.return_value = LanguagePreCommitResult( + mock_language_config_service.get_language_config.return_value = language.LanguagePreCommitResult( language="Python", version="abc123", - linter_config=LoadLinterConfigsResult(successful=False, linter_data=list()), + linter_config=language.LoadLinterConfigsResult( + successful=False, linter_data=list() + ), config_data=""" repos: - repo: http://sample-repo.com/goodie-finder # does not match our secrets_detectors @@ -179,13 +175,15 @@ def mock_loader_side_effect(resource): def test_that_language_support_calculates_a_serializable_hook_configuration( - language_support_service: LanguageSupportService, - mock_language_config_service: LanguageConfigService, + language_support_service: language_support.LanguageSupportService, + mock_language_config_service: language_config.LanguageConfigService, ): - mock_language_config_service.get_language_config.return_value = LanguagePreCommitResult( + mock_language_config_service.get_language_config.return_value = language.LanguagePreCommitResult( language="Python", version="abc123", - linter_config=LoadLinterConfigsResult(successful=False, linter_data=list()), + linter_config=language.LoadLinterConfigsResult( + successful=False, linter_data=list() + ), config_data=""" repos: - repo: http://sample-repo.com/hooks @@ -204,7 +202,7 @@ def test_that_language_support_calculates_a_serializable_hook_configuration( def test_that_language_support_does_not_identify_a_security_hook_if_config_uses_matching_repo_but_not_matching_hook( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_data_loader: MagicMock, mock_language_config_service: MagicMock, ): @@ -214,10 +212,12 @@ def mock_loader_side_effect(resource): - baddie-finder-hook """ - mock_language_config_service.get_language_config.return_value = LanguagePreCommitResult( + mock_language_config_service.get_language_config.return_value = language.LanguagePreCommitResult( language="Python", version="abc123", - linter_config=LoadLinterConfigsResult(successful=False, linter_data=list()), + linter_config=language.LoadLinterConfigsResult( + successful=False, linter_data=list() + ), config_data=""" repos: - repo: http://sample-repo.com/baddie-finder @@ -235,7 +235,7 @@ def mock_loader_side_effect(resource): # #### _write_pre_commit_configs #### def test_that_language_support_writes_linter_config_files( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_language_config_service: MagicMock, mock_data_loader: MagicMock, mock_open: MagicMock, @@ -247,10 +247,10 @@ def mock_loader_side_effect(resource): - baddie-finder """ - mock_language_config_service.get_language_config.return_value = LanguagePreCommitResult( + mock_language_config_service.get_language_config.return_value = language.LanguagePreCommitResult( language="Python", version="abc123", - linter_config=LoadLinterConfigsResult( + linter_config=language.LoadLinterConfigsResult( successful=True, linter_data=[{"filename": "test.txt", "settings": {}}], ), @@ -279,14 +279,14 @@ def mock_loader_side_effect(resource): def test_that_language_support_throws_exception_when_language_config_file_cannot_be_opened( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_language_config_service: MagicMock, mock_open: MagicMock, ): - mock_language_config_service.get_language_config.return_value = LanguagePreCommitResult( + mock_language_config_service.get_language_config.return_value = language.LanguagePreCommitResult( language="Python", version="abc123", - linter_config=LoadLinterConfigsResult( + linter_config=language.LoadLinterConfigsResult( successful=True, linter_data=[{"filename": "test.txt", "settings": {}}], ), @@ -314,15 +314,15 @@ def test_that_language_support_throws_exception_when_language_config_file_cannot def test_that_language_support_handles_invalid_language_config( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_language_config_service: MagicMock, mock_open: MagicMock, ): mock_language_config_service.get_language_config.return_value = ( - LanguagePreCommitResult( + language.LanguagePreCommitResult( language="Python", version="abc123", - linter_config=LoadLinterConfigsResult( + linter_config=language.LoadLinterConfigsResult( successful=True, linter_data=[{"filename": "test.txt", "settings": {}}], ), @@ -344,14 +344,14 @@ def test_that_language_support_handles_invalid_language_config( def test_that_language_support_handles_empty_repos_list( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_language_config_service: MagicMock, mock_data_loader: MagicMock, ): - mock_language_config_service.get_language_config.return_value = LanguagePreCommitResult( + mock_language_config_service.get_language_config.return_value = language.LanguagePreCommitResult( language="Python", version="abc123", - linter_config=LoadLinterConfigsResult( + linter_config=language.LoadLinterConfigsResult( successful=True, linter_data=[{"filename": "test.txt", "settings": {}}], ), @@ -373,7 +373,7 @@ def test_that_language_support_handles_empty_repos_list( def test_write_pre_commit_configs_writes_successfully( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_open: MagicMock, ): configs = [ @@ -403,7 +403,7 @@ def test_write_pre_commit_configs_writes_successfully( def test_build_pre_commit_displays_error_parsing_existing_config( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_language_config_service: MagicMock, mock_open: MagicMock, mock_echo: MagicMock, @@ -412,10 +412,10 @@ def test_build_pre_commit_displays_error_parsing_existing_config( patch("builtins.open", mock_open(read_data="data")), patch.object(yaml, "safe_load", side_effect=yaml.YAMLError), ): - mock_language_config_service.get_language_config.return_value = LanguagePreCommitResult( + mock_language_config_service.get_language_config.return_value = language.LanguagePreCommitResult( language="Python", version="abc123", - linter_config=LoadLinterConfigsResult( + linter_config=language.LoadLinterConfigsResult( successful=True, linter_data=[{"filename": "test.txt", "settings": {}}], ), @@ -438,14 +438,14 @@ def test_build_pre_commit_displays_error_parsing_existing_config( def test_build_pre_commit_respects_existing_pre_commit_config( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_language_config_service: MagicMock, mock_open: MagicMock, ): - mock_language_config_service.get_language_config.return_value = LanguagePreCommitResult( + mock_language_config_service.get_language_config.return_value = language.LanguagePreCommitResult( language="Python", version="abc123", - linter_config=LoadLinterConfigsResult( + linter_config=language.LoadLinterConfigsResult( successful=True, linter_data=[{"filename": "test.txt", "settings": {}}], ), @@ -476,7 +476,7 @@ def test_build_pre_commit_respects_existing_pre_commit_config( def test_write_pre_commit_configs_ignores_empty_linter_arr( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_open: MagicMock, ): language_support_service._write_pre_commit_configs([]) @@ -486,7 +486,7 @@ def test_write_pre_commit_configs_ignores_empty_linter_arr( def test_write_pre_commit_configs_returns_error_messages( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_open: MagicMock, ): mock_open.side_effect = Exception("error") @@ -503,7 +503,7 @@ def test_write_pre_commit_configs_returns_error_messages( mock_open.assert_called_once() mock_open.return_value.write.assert_not_called() - assert result == LinterConfigWriteResult( + assert result == language.LinterConfigWriteResult( error_messages=[ f"Failed to write {mock_filename} linter config file for {mock_language}" ], @@ -512,14 +512,14 @@ def test_write_pre_commit_configs_returns_error_messages( def test_write_pre_commit_configs_handles_empty_lint_configs( - language_support_service: LanguageSupportService, + language_support_service: language_support.LanguageSupportService, mock_open: MagicMock, ): result = language_support_service._write_pre_commit_configs([]) mock_open.assert_not_called() mock_open.return_value.write.assert_not_called() - assert result == LinterConfigWriteResult( + assert result == language.LinterConfigWriteResult( error_messages=[], successful_languages=[], ) diff --git a/tests/services/test_logging_service.py b/tests/modules/observability/test_logging_service.py similarity index 82% rename from tests/services/test_logging_service.py rename to tests/modules/observability/test_logging_service.py index a873ddda..23cfe4cf 100644 --- a/tests/services/test_logging_service.py +++ b/tests/modules/observability/test_logging_service.py @@ -3,10 +3,11 @@ import pytest from pytest_mock import MockerFixture +from secureli.modules.shared.models.config import HookConfiguration +from secureli.modules.shared.models.logging import LogAction from secureli.repositories.secureli_config import SecureliConfig -from secureli.services.logging import LoggingService, LogAction -from secureli.services.language_support import HookConfiguration +from secureli.modules.observability.observability_services import logging @pytest.fixture() @@ -17,7 +18,9 @@ def mock_path(mocker: MockerFixture) -> MagicMock: mock_path_instance = MagicMock() mock_path_instance.__truediv__.return_value = mock_file_path - mock_path_class = mocker.patch("secureli.services.logging.Path") + mock_path_class = mocker.patch( + "secureli.modules.observability.observability_services.logging.Path" + ) mock_path_class.return_value = mock_path_instance return mock_file_path @@ -42,15 +45,15 @@ def mock_language_support() -> MagicMock: @pytest.fixture() def logging_service( mock_language_support: MagicMock, mock_secureli_config: MagicMock -) -> LoggingService: - return LoggingService( +) -> logging.LoggingService: + return logging.LoggingService( language_support=mock_language_support, secureli_config=mock_secureli_config, ) def test_that_logging_service_success_creates_logs_folder_if_not_exists( - logging_service: LoggingService, + logging_service: logging.LoggingService, mock_path: MagicMock, mock_open: MagicMock, mock_secureli_config: MagicMock, @@ -66,7 +69,7 @@ def test_that_logging_service_success_creates_logs_folder_if_not_exists( def test_that_logging_service_failure_creates_logs_folder_if_not_exists( - logging_service: LoggingService, + logging_service: logging.LoggingService, mock_path: MagicMock, mock_open: MagicMock, mock_secureli_config: MagicMock, @@ -81,7 +84,7 @@ def test_that_logging_service_failure_creates_logs_folder_if_not_exists( def test_that_logging_service_success_logs_none_for_hook_config_if_not_initialized( - logging_service: LoggingService, + logging_service: logging.LoggingService, mock_path: MagicMock, mock_open: MagicMock, mock_secureli_config: MagicMock, diff --git a/tests/modules/pii_scanner/test_pii_scanner_service.py b/tests/modules/pii_scanner/test_pii_scanner_service.py new file mode 100644 index 00000000..36761cb7 --- /dev/null +++ b/tests/modules/pii_scanner/test_pii_scanner_service.py @@ -0,0 +1,129 @@ +import pytest +from pytest_mock import MockerFixture +from unittest.mock import MagicMock, Mock +import builtins +import contextlib, io +from pathlib import Path +from secureli.modules.pii_scanner.pii_scanner import PiiScannerService +from secureli.modules.shared.models.scan import ScanMode + + +test_folder_path = Path(".") + + +@pytest.fixture() +def mock_repo_files_repository() -> MagicMock: + mock_repo_files_repository = MagicMock() + mock_repo_files_repository.list_staged_files.return_value = ["fake_file_path"] + mock_repo_files_repository.list_repo_files.return_value = ["fake_file_path"] + return mock_repo_files_repository + + +@pytest.fixture() +def mock_echo() -> MagicMock: + mock_echo = MagicMock() + return mock_echo + + +@pytest.fixture() +def mock_open_fn(mocker: MockerFixture) -> MagicMock: + # The below data wouldn't ACTUALLY count as PII, but using fake PII here would prevent this code + # from being committed (as seCureLi scans itself before commit!) + # Instead, we mock the regex search function to pretend we found a PII match so we can assert the + # scanner's behavior + mock_open = mocker.mock_open( + read_data=""" + fake_email='pantsATpants.com' + fake_phone='phone-num-here' + """ + ) + return mocker.patch("builtins.open", mock_open) + + +# Include the below for any tests where you want PII to be "found" +@pytest.fixture() +def mock_re(mocker: MockerFixture) -> MagicMock: + match_object = mocker.patch("re.Match", lambda *args: True) + return mocker.patch("re.search", match_object) + + +@pytest.fixture() +def pii_scanner_service( + mock_repo_files_repository: MagicMock, mock_echo: MagicMock +) -> PiiScannerService: + return PiiScannerService(mock_repo_files_repository, mock_echo) + + +def test_that_pii_scanner_service_finds_potential_pii( + pii_scanner_service: PiiScannerService, + mock_repo_files_repository: MagicMock, + mock_open_fn: MagicMock, + mock_re: MagicMock, +): + scan_result = pii_scanner_service.scan_repo(test_folder_path, ScanMode.STAGED_ONLY) + + mock_repo_files_repository.list_staged_files.assert_called_once() + + assert scan_result.successful == False + assert len(scan_result.failures) == 1 + assert "Email" in scan_result.output + assert "Phone number" in scan_result.output + + +def test_that_pii_scanner_service_scans_all_files_when_specified( + pii_scanner_service: PiiScannerService, + mock_repo_files_repository: MagicMock, + mock_open_fn: MagicMock, +): + pii_scanner_service.scan_repo(test_folder_path, ScanMode.ALL_FILES) + + mock_repo_files_repository.list_repo_files.assert_called_once() + + +def test_that_pii_scanner_service_ignores_excluded_file_extensions( + pii_scanner_service: PiiScannerService, + mock_repo_files_repository: MagicMock, + mock_open_fn: MagicMock, + mock_re: MagicMock, +): + mock_repo_files_repository.list_staged_files.return_value = ["fake_file_path.md"] + + scan_result = pii_scanner_service.scan_repo(test_folder_path, ScanMode.STAGED_ONLY) + + assert scan_result.successful == True + + +def test_that_pii_scanner_service_only_scans_specific_files_if_provided( + pii_scanner_service: PiiScannerService, + mock_repo_files_repository: MagicMock, + mock_open_fn: MagicMock, + mock_re: MagicMock, +): + specified_file = "fake_file_path" + ignored_file = "not-the-file-we-want" + mock_repo_files_repository.list_staged_files.return_value = [ + specified_file, + ignored_file, + ] + scan_result = pii_scanner_service.scan_repo( + test_folder_path, ScanMode.STAGED_ONLY, [specified_file] + ) + + assert scan_result.successful == False + assert len(scan_result.failures) == 1 + assert scan_result.failures[0].file == specified_file + + +def test_that_pii_scanner_prints_when_exceptions_encountered( + pii_scanner_service: PiiScannerService, + mock_open_fn: MagicMock, + mock_echo: MagicMock, +): + mock_open_fn.side_effect = Exception("Oh no") + pii_scanner_service.scan_repo( + test_folder_path, + ScanMode.STAGED_ONLY, + ) + + mock_echo.print.assert_called_once() + assert "Error PII scanning" in mock_echo.print.call_args.args[0] diff --git a/tests/utilities/__init__.py b/tests/modules/shared/abstractions/__init__.py similarity index 100% rename from tests/utilities/__init__.py rename to tests/modules/shared/abstractions/__init__.py diff --git a/tests/abstractions/test_pre_commit.py b/tests/modules/shared/abstractions/test_pre_commit.py similarity index 89% rename from tests/abstractions/test_pre_commit.py rename to tests/modules/shared/abstractions/test_pre_commit.py index 7d7f7ce0..2dc712bf 100644 --- a/tests/abstractions/test_pre_commit.py +++ b/tests/modules/shared/abstractions/test_pre_commit.py @@ -9,16 +9,12 @@ import pytest from pytest_mock import MockerFixture -from secureli.abstractions.pre_commit import ( +from secureli.modules.shared.abstractions.pre_commit import ( InstallResult, PreCommitAbstraction, ) -from secureli.abstractions.echo import EchoAbstraction -from secureli.repositories.settings import ( - PreCommitSettings, - PreCommitRepo, - PreCommitHook, -) +from secureli.repositories import repo_settings + test_folder_path = Path("does-not-matter") example_git_sha = "a" * 40 @@ -26,13 +22,13 @@ @pytest.fixture() def settings_dict() -> dict: - return PreCommitSettings( + return repo_settings.PreCommitSettings( repos=[ - PreCommitRepo( + repo_settings.PreCommitRepo( repo="http://example-repo.com/", rev="master", hooks=[ - PreCommitHook( + repo_settings.PreCommitHook( id="hook-id", arguments=None, additional_args=None, @@ -49,7 +45,7 @@ def mock_hashlib(mocker: MockerFixture) -> MagicMock: mock_md5 = MagicMock() mock_hashlib.md5.return_value = mock_md5 mock_md5.hexdigest.return_value = "mock-hash-code" - mocker.patch("secureli.utilities.hash.hashlib", mock_hashlib) + mocker.patch("secureli.modules.shared.utilities.hashlib", mock_hashlib) return mock_hashlib @@ -59,7 +55,7 @@ def mock_hashlib_no_match(mocker: MockerFixture) -> MagicMock: mock_md5 = MagicMock() mock_hashlib.md5.return_value = mock_md5 mock_md5.hexdigest.side_effect = ["first-hash-code", "second-hash-code"] - mocker.patch("secureli.utilities.hash.hashlib", mock_hashlib) + mocker.patch("secureli.modules.shared.utilities.hashlib", mock_hashlib) return mock_hashlib @@ -74,7 +70,9 @@ def mock_data_loader() -> MagicMock: def mock_subprocess(mocker: MockerFixture) -> MagicMock: mock_subprocess = MagicMock() mock_subprocess.run.return_value = CompletedProcess(args=[], returncode=0) - mocker.patch("secureli.abstractions.pre_commit.subprocess", mock_subprocess) + mocker.patch( + "secureli.modules.shared.abstractions.pre_commit.subprocess", mock_subprocess + ) return mock_subprocess @@ -432,14 +430,16 @@ def test_check_for_hook_updates_infers_freeze_param_when_not_provided( rev_is_sha: bool, ): with um.patch( - "secureli.abstractions.pre_commit.HookRepoRevInfo.from_config" + "secureli.modules.shared.abstractions.pre_commit.HookRepoRevInfo.from_config" ) as mock_hook_repo_rev_info: - pre_commit_config_repo = PreCommitRepo( + pre_commit_config_repo = repo_settings.PreCommitRepo( repo="http://example-repo.com/", rev=rev, - hooks=[PreCommitHook(id="hook-id")], + hooks=[repo_settings.PreCommitHook(id="hook-id")], + ) + pre_commit_config = repo_settings.PreCommitSettings( + repos=[pre_commit_config_repo] ) - pre_commit_config = PreCommitSettings(repos=[pre_commit_config_repo]) rev_info_mock = MagicMock(rev=pre_commit_config_repo.rev) mock_hook_repo_rev_info.return_value = rev_info_mock rev_info_mock.update.return_value = rev_info_mock # Returning the same revision info on update means the hook will be considered up to date @@ -455,14 +455,16 @@ def test_check_for_hook_updates_respects_freeze_param_when_false( regardless of whether the existing rev is a tag or a commit hash. """ with um.patch( - "secureli.abstractions.pre_commit.HookRepoRevInfo.from_config" + "secureli.modules.shared.abstractions.pre_commit.HookRepoRevInfo.from_config" ) as mock_hook_repo_rev_info: - pre_commit_config_repo = PreCommitRepo( + pre_commit_config_repo = repo_settings.PreCommitRepo( repo="http://example-repo.com/", rev=example_git_sha, - hooks=[PreCommitHook(id="hook-id")], + hooks=[repo_settings.PreCommitHook(id="hook-id")], + ) + pre_commit_config = repo_settings.PreCommitSettings( + repos=[pre_commit_config_repo] ) - pre_commit_config = PreCommitSettings(repos=[pre_commit_config_repo]) rev_info_mock = MagicMock(rev=pre_commit_config_repo.rev) mock_hook_repo_rev_info.return_value = rev_info_mock rev_info_mock.update.return_value = rev_info_mock # Returning the same revision info on update means the hook will be considered up to date @@ -474,14 +476,16 @@ def test_check_for_hook_updates_respects_freeze_param_when_true( pre_commit: PreCommitAbstraction, ): with um.patch( - "secureli.abstractions.pre_commit.HookRepoRevInfo.from_config" + "secureli.modules.shared.abstractions.pre_commit.HookRepoRevInfo.from_config" ) as mock_hook_repo_rev_info: - pre_commit_config_repo = PreCommitRepo( + pre_commit_config_repo = repo_settings.PreCommitRepo( repo="http://example-repo.com/", rev="tag1", - hooks=[PreCommitHook(id="hook-id")], + hooks=[repo_settings.PreCommitHook(id="hook-id")], + ) + pre_commit_config = repo_settings.PreCommitSettings( + repos=[pre_commit_config_repo] ) - pre_commit_config = PreCommitSettings(repos=[pre_commit_config_repo]) rev_info_mock = MagicMock(rev=pre_commit_config_repo.rev) mock_hook_repo_rev_info.return_value = rev_info_mock rev_info_mock.update.return_value = rev_info_mock # Returning the same revision info on update means the hook will be considered up to date @@ -493,15 +497,17 @@ def test_check_for_hook_updates_returns_repos_with_new_revs( pre_commit: PreCommitAbstraction, ): with um.patch( - "secureli.abstractions.pre_commit.HookRepoRevInfo" + "secureli.modules.shared.abstractions.pre_commit.HookRepoRevInfo" ) as mock_hook_repo_rev_info: repo_urls = ["http://example-repo.com/", "http://example-repo-2.com/"] old_rev = "tag1" repo_1_new_rev = "tag2" - pre_commit_config = PreCommitSettings( + pre_commit_config = repo_settings.PreCommitSettings( repos=[ - PreCommitRepo( - repo=repo_url, rev=old_rev, hooks=[PreCommitHook(id="hook-id")] + repo_settings.PreCommitRepo( + repo=repo_url, + rev=old_rev, + hooks=[repo_settings.PreCommitHook(id="hook-id")], ) for repo_url in repo_urls ] @@ -565,3 +571,15 @@ def test_migrate_config_file_moves_pre_commit_conig( f"Moving {old_location} to {new_location}..." ) mock_move.assert_called_once() + + +def test_get_pre_commit_config_path_is_correct_returns_expected_values( + pre_commit: PreCommitAbstraction, +): + bad_result = pre_commit.get_pre_commit_config_path_is_correct(test_folder_path) + assert bad_result == False + with (um.patch.object(Path, "exists", return_value=True),): + good_result = pre_commit.get_pre_commit_config_path_is_correct( + test_folder_path / ".secureli" + ) + assert good_result == True diff --git a/tests/abstractions/test_pygments_lexer_guesser.py b/tests/modules/shared/abstractions/test_pygments_lexer_guesser.py similarity index 90% rename from tests/abstractions/test_pygments_lexer_guesser.py rename to tests/modules/shared/abstractions/test_pygments_lexer_guesser.py index c97531d3..2aa4eff8 100644 --- a/tests/abstractions/test_pygments_lexer_guesser.py +++ b/tests/modules/shared/abstractions/test_pygments_lexer_guesser.py @@ -4,7 +4,7 @@ import pytest from pytest_mock import MockerFixture -from secureli.abstractions.lexer_guesser import PygmentsLexerGuesser +from secureli.modules.shared.abstractions.lexer_guesser import PygmentsLexerGuesser @pytest.fixture() diff --git a/tests/modules/shared/abstractions/test_repo.py b/tests/modules/shared/abstractions/test_repo.py new file mode 100644 index 00000000..5a99462d --- /dev/null +++ b/tests/modules/shared/abstractions/test_repo.py @@ -0,0 +1,25 @@ +from pathlib import Path +from unittest.mock import MagicMock +import pytest +from pytest_mock import MockerFixture + +from secureli.modules.shared.abstractions.repo import GitRepo + + +@pytest.fixture() +def git_repo() -> GitRepo: + return GitRepo() + + +@pytest.fixture() +def mock_repo(mocker: MockerFixture) -> MagicMock: + return mocker.patch("git.Repo", MagicMock()) + + +def test_that_get_commit_diff_returns_file_diff( + git_repo: GitRepo, mock_repo: MagicMock +): + mock_files = [Path("test_file.py"), Path("test_file2.py")] + mock_repo.return_value.head.commit.diff.return_value = mock_files + result = git_repo.get_commit_diff() + assert result is mock_files diff --git a/tests/abstractions/test_typer_echo.py b/tests/modules/shared/abstractions/test_typer_echo.py similarity index 87% rename from tests/abstractions/test_typer_echo.py rename to tests/modules/shared/abstractions/test_typer_echo.py index 4967f4de..6c089ab2 100644 --- a/tests/abstractions/test_typer_echo.py +++ b/tests/modules/shared/abstractions/test_typer_echo.py @@ -1,11 +1,9 @@ from pytest_mock import MockerFixture -from secureli.abstractions.echo import TyperEcho +from secureli.modules.shared.abstractions.echo import TyperEcho from unittest.mock import MagicMock, ANY import pytest -from secureli.models.echo import Color - -from secureli.utilities.logging import EchoLevel +from secureli.modules.shared.models.echo import Color, Level ECHO_SECURELI_PREFIX = "[seCureLI]" @@ -48,7 +46,7 @@ def typer_echo(request) -> TyperEcho: @pytest.mark.parametrize( "typer_echo", - [EchoLevel.info, EchoLevel.debug, EchoLevel.error, EchoLevel.warn], + [Level.info, Level.debug, Level.error, Level.warn], indirect=True, ) def test_that_typer_echo_renders_print_messages_correctly( @@ -68,10 +66,10 @@ def test_that_typer_echo_renders_print_messages_correctly( @pytest.mark.parametrize( "typer_echo", [ - EchoLevel.debug, - EchoLevel.info, - EchoLevel.warn, - EchoLevel.error, + Level.debug, + Level.info, + Level.warn, + Level.error, ], indirect=True, ) @@ -90,7 +88,7 @@ def test_that_typer_echo_renders_errors_correctly( @pytest.mark.parametrize( - "typer_echo", [EchoLevel.warn, EchoLevel.info, EchoLevel.debug], indirect=True + "typer_echo", [Level.warn, Level.info, Level.debug], indirect=True ) def test_that_typer_echo_renders_warnings_correctly( typer_echo: TyperEcho, @@ -106,7 +104,7 @@ def test_that_typer_echo_renders_warnings_correctly( ) -@pytest.mark.parametrize("typer_echo", [EchoLevel.info, EchoLevel.debug], indirect=True) +@pytest.mark.parametrize("typer_echo", [Level.info, Level.debug], indirect=True) def test_that_typer_echo_renders_info_correctly( typer_echo: TyperEcho, mock_echo_text: str, @@ -123,7 +121,7 @@ def test_that_typer_echo_renders_info_correctly( @pytest.mark.parametrize( "typer_echo", - [EchoLevel.debug], + [Level.debug], indirect=True, ) def test_that_typer_echo_renders_debug_messages_correctly( @@ -139,7 +137,7 @@ def test_that_typer_echo_renders_debug_messages_correctly( @pytest.mark.parametrize( "typer_echo", - [EchoLevel.off], + [Level.off], indirect=True, ) def test_that_typer_echo_suppresses_all_messages_when_off( @@ -158,7 +156,7 @@ def test_that_typer_echo_suppresses_all_messages_when_off( @pytest.mark.parametrize( "typer_echo", - [EchoLevel.off], + [Level.off], indirect=True, ) def test_that_typer_echo_suppresses_error_messages( @@ -170,7 +168,7 @@ def test_that_typer_echo_suppresses_error_messages( @pytest.mark.parametrize( "typer_echo", - [EchoLevel.error, EchoLevel.off], + [Level.error, Level.off], indirect=True, ) def test_that_typer_echo_suppresses_warning_messages( @@ -182,7 +180,7 @@ def test_that_typer_echo_suppresses_warning_messages( @pytest.mark.parametrize( "typer_echo", - [EchoLevel.warn, EchoLevel.error, EchoLevel.off], + [Level.warn, Level.error, Level.off], indirect=True, ) def test_that_typer_echo_suppresses_info_messages( @@ -194,7 +192,7 @@ def test_that_typer_echo_suppresses_info_messages( @pytest.mark.parametrize( "typer_echo", - [EchoLevel.info, EchoLevel.warn, EchoLevel.error, EchoLevel.off], + [Level.info, Level.warn, Level.error, Level.off], indirect=True, ) def test_that_typer_echo_suppresses_debug_messages( @@ -206,7 +204,7 @@ def test_that_typer_echo_suppresses_debug_messages( @pytest.mark.parametrize( "typer_echo", - [EchoLevel.info], + [Level.info], indirect=True, ) def test_that_typer_echo_prompts_user_for_confirmation( @@ -220,7 +218,7 @@ def test_that_typer_echo_prompts_user_for_confirmation( def test_that_typer_echo_implements_prompt_with_default(mock_typer_prompt: MagicMock): - typer_echo = TyperEcho(level=EchoLevel.info) + typer_echo = TyperEcho(level=Level.info) message = "test message" default_response = "default user response" typer_echo.prompt(message=message, default_response=default_response) @@ -233,7 +231,7 @@ def test_that_typer_echo_implements_prompt_with_default(mock_typer_prompt: Magic def test_that_typer_echo_implements_prompt_without_default( mock_typer_prompt: MagicMock, ): - typer_echo = TyperEcho(level=EchoLevel.info) + typer_echo = TyperEcho(level=Level.info) message = "test message" typer_echo.prompt(message=message) diff --git a/tests/modules/shared/resources/__init__.py b/tests/modules/shared/resources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/resources/test_read_resource.py b/tests/modules/shared/resources/test_read_resource.py similarity index 89% rename from tests/resources/test_read_resource.py rename to tests/modules/shared/resources/test_read_resource.py index 72df5a1b..da1b8750 100644 --- a/tests/resources/test_read_resource.py +++ b/tests/modules/shared/resources/test_read_resource.py @@ -4,7 +4,7 @@ import pytest from pytest_mock import MockerFixture -import secureli.resources +import secureli.modules.shared.resources @pytest.fixture() @@ -14,7 +14,7 @@ def mock_open_resource(mocker: MockerFixture) -> MagicMock: @pytest.fixture() def read_resource(mock_open_resource: MagicMock) -> Callable[[str], str]: - return secureli.resources.read_resource + return secureli.modules.shared.resources.read_resource def test_that_read_resource_opens_specified_file_in_files_folder( diff --git a/tests/modules/shared/utilities/__init__.py b/tests/modules/shared/utilities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/utilities/test_formatter.py b/tests/modules/shared/utilities/test_formatter.py similarity index 90% rename from tests/utilities/test_formatter.py rename to tests/modules/shared/utilities/test_formatter.py index c2e711f6..a1979e60 100644 --- a/tests/utilities/test_formatter.py +++ b/tests/modules/shared/utilities/test_formatter.py @@ -1,4 +1,4 @@ -from secureli.utilities.formatter import format_sentence_list +from secureli.modules.shared.utilities import format_sentence_list def test_format_sentence_list_handles_missing_list(): diff --git a/tests/utilities/test_git_meta.py b/tests/modules/shared/utilities/test_git_meta.py similarity index 74% rename from tests/utilities/test_git_meta.py rename to tests/modules/shared/utilities/test_git_meta.py index 738c6bc6..59d4eb2c 100644 --- a/tests/utilities/test_git_meta.py +++ b/tests/modules/shared/utilities/test_git_meta.py @@ -4,23 +4,27 @@ import pytest from pytest_mock import MockerFixture -from secureli.utilities.git_meta import git_user_email, origin_url, current_branch_name +from secureli.modules.shared import utilities -mock_git_origin_url = r"git@github.com:my-org/repo%20with%20spaces.git" +mock_git_origin_url = ( + r"git@github.com:my-org/repo%20with%20spaces.git" # disable-pii-scan +) @pytest.fixture() def mock_subprocess(mocker: MockerFixture) -> MagicMock: - mock_subprocess = mocker.patch("secureli.utilities.git_meta.subprocess") + mock_subprocess = mocker.patch("secureli.modules.shared.utilities.subprocess") mock_subprocess.run.return_value = CompletedProcess( - args=[], returncode=0, stdout="great.engineer@slalom.com\n".encode("utf8") + args=[], + returncode=0, + stdout="great.engineer@slalom.com\n".encode("utf8"), # disable-pii-scan ) return mock_subprocess @pytest.fixture() def mock_configparser(mocker: MockerFixture) -> MagicMock: - mock_configparser = mocker.patch("secureli.utilities.git_meta.configparser") + mock_configparser = mocker.patch("secureli.modules.shared.utilities.configparser") mock_configparser_instance = MagicMock() mock_configparser_instance['remote "origin"'].get.return_value = ( "https://fake-build.com/git/repo" @@ -57,14 +61,16 @@ def mock_open_io_error(mocker: MockerFixture) -> MagicMock: def test_git_user_email_loads_user_email_via_git_subprocess(mock_subprocess: MagicMock): - result = git_user_email() + result = utilities.git_user_email() mock_subprocess.run.assert_called_once() - assert result == "great.engineer@slalom.com" # note: without trailing newline + assert ( + result == "great.engineer@slalom.com" # disable-pii-scan + ) # note: without trailing newline def test_origin_url_parses_config_to_get_origin_url(mock_configparser: MagicMock): - result = origin_url() + result = utilities.origin_url() mock_configparser.read.assert_called_once_with(".git/config") assert result == "https://fake-build.com/git/repo" @@ -73,7 +79,7 @@ def test_origin_url_parses_config_to_get_origin_url(mock_configparser: MagicMock def test_current_branch_name_finds_ref_name_from_head_file( mock_open_git_head: MagicMock, ): - result = current_branch_name() + result = utilities.current_branch_name() assert result == "feature/wicked-sick-branch" @@ -81,10 +87,10 @@ def test_current_branch_name_finds_ref_name_from_head_file( def test_current_branch_name_yields_unknown_due_to_io_error( mock_open_io_error: MagicMock, ): - result = current_branch_name() + result = utilities.current_branch_name() assert result == "UNKNOWN" def test_configparser_can_read_origin_url_with_percent(mock_open_git_origin: MagicMock): - assert origin_url() == mock_git_origin_url + assert utilities.origin_url() == mock_git_origin_url diff --git a/tests/utilities/test_logging.py b/tests/modules/shared/utilities/test_logging.py similarity index 64% rename from tests/utilities/test_logging.py rename to tests/modules/shared/utilities/test_logging.py index 210fd85e..bfbf5706 100644 --- a/tests/utilities/test_logging.py +++ b/tests/modules/shared/utilities/test_logging.py @@ -1,13 +1,13 @@ -from secureli.utilities.logging import EchoLevel +from secureli.modules.shared.models.echo import Level def test_that_echo_level_str_returns_enum_val(): - level = EchoLevel.info + level = Level.info assert str(level) == level.value def test_that_echo_level_repr_returns_str_implementation(): - level = EchoLevel.debug + level = Level.debug assert repr(level) == str(level) diff --git a/tests/utilities/test_patterns.py b/tests/modules/shared/utilities/test_patterns.py similarity index 92% rename from tests/utilities/test_patterns.py rename to tests/modules/shared/utilities/test_patterns.py index 505f58d1..558839f4 100644 --- a/tests/utilities/test_patterns.py +++ b/tests/modules/shared/utilities/test_patterns.py @@ -1,4 +1,4 @@ -from secureli.utilities.patterns import combine_patterns +from secureli.modules.shared.utilities import combine_patterns def test_that_combine_patterns_returns_none_for_empty_list(): diff --git a/tests/utilities/test_secureli_meta.py b/tests/modules/shared/utilities/test_secureli_meta.py similarity index 84% rename from tests/utilities/test_secureli_meta.py rename to tests/modules/shared/utilities/test_secureli_meta.py index 86392072..17f0ee42 100644 --- a/tests/utilities/test_secureli_meta.py +++ b/tests/modules/shared/utilities/test_secureli_meta.py @@ -17,7 +17,9 @@ def mock_open_repo_config(mocker: MockerFixture) -> MagicMock: def mock_secureli_meta_path(mocker: MockerFixture) -> MagicMock: mock_path_instance = MagicMock() - mock_path_class = mocker.patch("secureli.utilities.secureli_meta.Path") + mock_path_class = mocker.patch( + "secureli.modules.shared.utilities.secureli_meta.Path" + ) mock_path_class.return_value = mock_path_instance return mock_path_instance diff --git a/tests/utilities/test_usage_stats.py b/tests/modules/shared/utilities/test_usage_stats.py similarity index 62% rename from tests/utilities/test_usage_stats.py rename to tests/modules/shared/utilities/test_usage_stats.py index 36cba76d..9f98e2c3 100644 --- a/tests/utilities/test_usage_stats.py +++ b/tests/modules/shared/utilities/test_usage_stats.py @@ -1,16 +1,10 @@ -from secureli.consts.logging import ( - TELEMETRY_ENDPOINT_ENV_VAR_NAME, - TELEMETRY_KEY_ENV_VAR_NAME, -) -from secureli.models.publish_results import PublishLogResult -from secureli.models.result import Result -from secureli.repositories.settings import TelemetrySettings +from secureli.modules.observability.consts import logging +from secureli.modules.shared.models.publish_results import PublishLogResult +from secureli.modules.shared.models.result import Result +from secureli.modules.shared.models.scan import ScanFailure +from secureli.repositories.repo_settings import TelemetrySettings from secureli.settings import Settings -from secureli.utilities.usage_stats import ( - post_log, - convert_failures_to_failure_count, -) -from secureli.services.scanner import Failure +from secureli.modules.shared import utilities from unittest import mock from unittest.mock import Mock, patch @@ -19,12 +13,12 @@ def test_that_convert_failures_to_failure_count_returns_correct_count(): list_of_failure = [ - Failure(id="testfailid1", file="testfile1", repo="testrepo1"), - Failure(id="testfailid1", file="testfile2", repo="testrepo1"), - Failure(id="testfailid2", file="testfile1", repo="testrepo1"), + ScanFailure(id="testfailid1", file="testfile1", repo="testrepo1"), + ScanFailure(id="testfailid1", file="testfile2", repo="testrepo1"), + ScanFailure(id="testfailid2", file="testfile1", repo="testrepo1"), ] - result = convert_failures_to_failure_count(list_of_failure) + result = utilities.convert_failures_to_failure_count(list_of_failure) assert result["testfailid1"] == 2 assert result["testfailid2"] == 1 @@ -33,7 +27,7 @@ def test_that_convert_failures_to_failure_count_returns_correct_count(): def test_that_convert_failures_to_failure_count_returns_correctly_when_no_failure(): list_of_failure = [] - result = convert_failures_to_failure_count(list_of_failure) + result = utilities.convert_failures_to_failure_count(list_of_failure) assert result == {} @@ -41,14 +35,14 @@ def test_that_convert_failures_to_failure_count_returns_correctly_when_no_failur @mock.patch.dict( os.environ, { - f"{TELEMETRY_ENDPOINT_ENV_VAR_NAME}": "testendpoint", - f"{TELEMETRY_KEY_ENV_VAR_NAME}": "", + f"{logging.TELEMETRY_ENDPOINT_ENV_VAR_NAME}": "testendpoint", + f"{logging.TELEMETRY_KEY_ENV_VAR_NAME}": "", }, clear=True, ) @patch("requests.post") def test_post_log_with_no_api_key(mock_requests): - result = post_log("testing", Settings()) + result = utilities.post_log("testing", Settings()) mock_requests.assert_not_called() @@ -62,14 +56,16 @@ def test_post_log_with_no_api_key(mock_requests): @mock.patch.dict( os.environ, { - f"{TELEMETRY_ENDPOINT_ENV_VAR_NAME}": "", - f"{TELEMETRY_KEY_ENV_VAR_NAME}": "testkey", + f"{logging.TELEMETRY_ENDPOINT_ENV_VAR_NAME}": "", + f"{logging.TELEMETRY_KEY_ENV_VAR_NAME}": "testkey", }, clear=True, ) @patch("requests.post") def test_post_log_with_no_api_endpoint(mock_requests): - result = post_log("testing", Settings(telemetry=TelemetrySettings(api_url=None))) + result = utilities.post_log( + "testing", Settings(telemetry=TelemetrySettings(api_url=None)) + ) mock_requests.assert_not_called() @@ -82,8 +78,8 @@ def test_post_log_with_no_api_endpoint(mock_requests): @mock.patch.dict( os.environ, { - f"{TELEMETRY_ENDPOINT_ENV_VAR_NAME}": "testendpoint", - f"{TELEMETRY_KEY_ENV_VAR_NAME}": "testkey", + f"{logging.TELEMETRY_ENDPOINT_ENV_VAR_NAME}": "testendpoint", + f"{logging.TELEMETRY_KEY_ENV_VAR_NAME}": "testkey", }, # pragma: allowlist secret clear=True, ) @@ -91,7 +87,7 @@ def test_post_log_with_no_api_endpoint(mock_requests): def test_post_log_http_error(mock_requests): mock_requests.side_effect = Exception("test exception") - result = post_log("test_log_data", Settings()) + result = utilities.post_log("test_log_data", Settings()) mock_requests.assert_called_once_with( url="testendpoint", headers={"Api-Key": "testkey"}, data="test_log_data" @@ -105,8 +101,8 @@ def test_post_log_http_error(mock_requests): @mock.patch.dict( os.environ, { - f"{TELEMETRY_ENDPOINT_ENV_VAR_NAME}": "testendpoint", - f"{TELEMETRY_KEY_ENV_VAR_NAME}": "testkey", + f"{logging.TELEMETRY_ENDPOINT_ENV_VAR_NAME}": "testendpoint", + f"{logging.TELEMETRY_KEY_ENV_VAR_NAME}": "testkey", }, # pragma: allowlist secret clear=True, ) @@ -114,7 +110,7 @@ def test_post_log_http_error(mock_requests): def test_post_log_happy_path(mock_requests): mock_requests.return_value = Mock(status_code=202, text="sample-response") - result = post_log("test_log_data", Settings()) + result = utilities.post_log("test_log_data", Settings()) mock_requests.assert_called_once_with( url="testendpoint", headers={"Api-Key": "testkey"}, data="test_log_data" @@ -127,8 +123,8 @@ def test_post_log_happy_path(mock_requests): @mock.patch.dict( os.environ, { - f"{TELEMETRY_ENDPOINT_ENV_VAR_NAME}": "", - f"{TELEMETRY_KEY_ENV_VAR_NAME}": "testkey", + f"{logging.TELEMETRY_ENDPOINT_ENV_VAR_NAME}": "", + f"{logging.TELEMETRY_KEY_ENV_VAR_NAME}": "testkey", }, # pragma: allowlist secret clear=True, ) @@ -136,7 +132,7 @@ def test_post_log_happy_path(mock_requests): def test_post_log_uses_settings_endpoint_if_no_env_endpoint(mock_requests): mock_requests.return_value = Mock(status_code=202, text="sample-response") - result = post_log( + result = utilities.post_log( "test_log_data", Settings(telemetry=TelemetrySettings(api_url="testendpoint")) ) diff --git a/tests/services/test_secureli_ignore.py b/tests/modules/test_secureli_ignore.py similarity index 62% rename from tests/services/test_secureli_ignore.py rename to tests/modules/test_secureli_ignore.py index fa9db074..609df736 100644 --- a/tests/services/test_secureli_ignore.py +++ b/tests/modules/test_secureli_ignore.py @@ -1,34 +1,30 @@ import pytest -from secureli.services.secureli_ignore import SecureliIgnoreService -from secureli.settings import ( - Settings, - RepoFilesSettings, - LanguageSupportSettings, - EchoSettings, -) +from secureli.modules.secureli_ignore import SecureliIgnoreService +from secureli.repositories import repo_settings +from secureli.settings import Settings @pytest.fixture() -def repo_files() -> RepoFilesSettings: - return RepoFilesSettings() +def repo_files() -> repo_settings.RepoFilesSettings: + return repo_settings.RepoFilesSettings() @pytest.fixture() -def echo() -> EchoSettings: - return EchoSettings() +def echo() -> repo_settings.EchoSettings: + return repo_settings.EchoSettings() @pytest.fixture() -def language_support() -> LanguageSupportSettings: - return LanguageSupportSettings() +def language_support() -> repo_settings.LanguageSupportSettings: + return repo_settings.LanguageSupportSettings() @pytest.fixture() def settings( - repo_files: RepoFilesSettings, - echo: EchoSettings, - language_support: LanguageSupportSettings, + repo_files: repo_settings.RepoFilesSettings, + echo: repo_settings.EchoSettings, + language_support: repo_settings.LanguageSupportSettings, ) -> Settings: return Settings( repo_files=repo_files, diff --git a/tests/repositories/test_repo_files_repository.py b/tests/repositories/test_repo_files_repository.py index 5a5ab450..4feee37d 100644 --- a/tests/repositories/test_repo_files_repository.py +++ b/tests/repositories/test_repo_files_repository.py @@ -3,10 +3,18 @@ import pytest from pytest_mock import MockerFixture +from subprocess import CompletedProcess from secureli.repositories.repo_files import RepoFilesRepository +@pytest.fixture() +def mock_subprocess(mocker: MockerFixture) -> MagicMock: + mock_subprocess = MagicMock() + mocker.patch("secureli.repositories.repo_files.subprocess", mock_subprocess) + return mock_subprocess + + @pytest.fixture() def git_not_exists_folder_path() -> MagicMock: git_folder_path = MagicMock() @@ -145,6 +153,22 @@ def test_that_list_repo_files_raises_value_error_without_git_repo( repo_files_repository.list_repo_files(git_not_exists_folder_path) +def test_that_list_staged_files_returns_list_of_staged_files( + repo_files_repository: RepoFilesRepository, + mock_subprocess: MagicMock, +): + fake_file_1 = "i/am/staged" + fake_file_2 = "also/staged" + mock_subprocess.run.return_value = CompletedProcess( + args=[], + returncode=0, + stdout=f"{fake_file_1}\n{fake_file_2}\n".encode("utf8"), + ) + + result = repo_files_repository.list_staged_files(Path(".")) + assert result == [fake_file_1, fake_file_2] + + def test_that_list_repo_files_raises_value_error_if_dot_git_is_a_file_somehow( repo_files_repository: RepoFilesRepository, git_a_file_for_some_reason_folder_path: MagicMock, diff --git a/tests/repositories/test_secureli_config.py b/tests/repositories/test_secureli_config.py index 0f3a4d84..106b928a 100644 --- a/tests/repositories/test_secureli_config.py +++ b/tests/repositories/test_secureli_config.py @@ -3,12 +3,7 @@ import pytest from pytest_mock import MockerFixture -import secureli.repositories.secureli_config as SecureliConfigAll -from secureli.repositories.secureli_config import ( - SecureliConfigRepository, - SecureliConfig, - VerifyConfigOutcome, -) +from secureli.repositories import secureli_config @pytest.fixture() @@ -86,25 +81,25 @@ def existent_path_old_schema(mocker: MockerFixture) -> MagicMock: @pytest.fixture() -def secureli_config() -> SecureliConfigRepository: - secureli_config = SecureliConfigRepository() - return secureli_config +def secureli_config_fixture() -> secureli_config.SecureliConfigRepository: + secureli_config_fixture = secureli_config.SecureliConfigRepository() + return secureli_config_fixture def test_that_repo_synthesizes_default_config_when_missing( non_existent_path: MagicMock, - secureli_config: SecureliConfigRepository, + secureli_config_fixture: secureli_config.SecureliConfigRepository, ): - config = secureli_config.load() + config = secureli_config_fixture.load() assert config.languages is None def test_that_repo_loads_config_when_present( existent_path: MagicMock, - secureli_config: SecureliConfigRepository, + secureli_config_fixture: secureli_config.SecureliConfigRepository, ): - config = secureli_config.load() + config = secureli_config_fixture.load() assert config.languages == ["RadLang"] @@ -112,49 +107,54 @@ def test_that_repo_loads_config_when_present( def test_that_repo_saves_config( existent_path: MagicMock, mock_open: MagicMock, - secureli_config: SecureliConfigRepository, + secureli_config_fixture: secureli_config.SecureliConfigRepository, ): - config = SecureliConfig(languages=["AwesomeLang"]) - secureli_config.save(config) + config = secureli_config.SecureliConfig(languages=["AwesomeLang"]) + secureli_config_fixture.save(config) mock_open.assert_called_once() def test_that_repo_validates_most_current_schema( - existent_path: MagicMock, secureli_config: SecureliConfigRepository + existent_path: MagicMock, + secureli_config_fixture: secureli_config.SecureliConfigRepository, ): - result = secureli_config.verify() + result = secureli_config_fixture.verify() - assert result == VerifyConfigOutcome.UP_TO_DATE + assert result == secureli_config.VerifyConfigOutcome.UP_TO_DATE def test_that_repo_catches_deprecated_schema( - existent_path_old_schema: MagicMock, secureli_config: SecureliConfigRepository + existent_path_old_schema: MagicMock, + secureli_config_fixture: secureli_config.SecureliConfigRepository, ): - result = secureli_config.verify() + result = secureli_config_fixture.verify() - assert result == VerifyConfigOutcome.OUT_OF_DATE + assert result == secureli_config.VerifyConfigOutcome.OUT_OF_DATE def test_that_repo_does_not_validate_with_missing_config( - non_existent_path: MagicMock, secureli_config: SecureliConfigRepository + non_existent_path: MagicMock, + secureli_config_fixture: secureli_config.SecureliConfigRepository, ): - result = secureli_config.verify() + result = secureli_config_fixture.verify() - assert result == VerifyConfigOutcome.MISSING + assert result == secureli_config.VerifyConfigOutcome.MISSING def test_that_repo_updates_config( - existent_path_old_schema: MagicMock, secureli_config: SecureliConfigRepository + existent_path_old_schema: MagicMock, + secureli_config_fixture: secureli_config.SecureliConfigRepository, ): - result = secureli_config.update() + result = secureli_config_fixture.update() assert result.languages def test_that_update_returns_empty_config_if_missing_config_file( - non_existent_path: MagicMock, secureli_config: SecureliConfigRepository + non_existent_path: MagicMock, + secureli_config_fixture: secureli_config.SecureliConfigRepository, ): - result = secureli_config.update() + result = secureli_config_fixture.update() - assert result == SecureliConfig() + assert result == secureli_config.SecureliConfig() diff --git a/tests/repositories/test_settings_repository.py b/tests/repositories/test_settings_repository.py index 8787a267..76997e6c 100644 --- a/tests/repositories/test_settings_repository.py +++ b/tests/repositories/test_settings_repository.py @@ -3,12 +3,8 @@ import pytest from pytest_mock import MockerFixture -from secureli.repositories.settings import ( - SecureliFile, - SecureliRepository, - EchoLevel, - EchoSettings, -) +from secureli.repositories import repo_settings +from secureli.modules.shared.models.echo import Level @pytest.fixture() @@ -22,7 +18,7 @@ def non_existent_path(mocker: MockerFixture) -> MagicMock: mock_path_class = MagicMock() mock_path_class.return_value = mock_folder_path - mocker.patch("secureli.repositories.settings.Path", mock_path_class) + mocker.patch("secureli.repositories.repo_settings.Path", mock_path_class) return mock_folder_path @@ -55,29 +51,29 @@ def existent_path(mocker: MockerFixture) -> MagicMock: ) mocker.patch("builtins.open", mock_open) - mocker.patch("secureli.repositories.settings.Path", mock_path_class) + mocker.patch("secureli.repositories.repo_settings.Path", mock_path_class) return mock_folder_path @pytest.fixture() -def settings_repository() -> SecureliRepository: - settings_repository = SecureliRepository() +def settings_repository() -> repo_settings.SecureliRepository: + settings_repository = repo_settings.SecureliRepository() return settings_repository def test_that_settings_file_loads_settings_when_present( existent_path: MagicMock, - settings_repository: SecureliRepository, + settings_repository: repo_settings.SecureliRepository, ): secureli_file = settings_repository.load(existent_path) - assert secureli_file.echo.level == EchoLevel.error + assert secureli_file.echo.level == Level.error def test_that_settings_file_created_when_not_present( non_existent_path: MagicMock, - settings_repository: SecureliRepository, + settings_repository: repo_settings.SecureliRepository, ): secureli_file = settings_repository.load(non_existent_path) @@ -87,10 +83,10 @@ def test_that_settings_file_created_when_not_present( def test_that_repo_saves_config( existent_path: MagicMock, mock_open: MagicMock, - settings_repository: SecureliRepository, + settings_repository: repo_settings.SecureliRepository, ): - echo_level = EchoSettings(level=EchoLevel.info) - settings_file = SecureliFile(echo=echo_level) + echo_level = repo_settings.EchoSettings(level=Level.info) + settings_file = repo_settings.SecureliFile(echo=echo_level) settings_repository.save(settings_file) mock_open.assert_called_once() @@ -99,9 +95,9 @@ def test_that_repo_saves_config( def test_that_repo_saves_without_echo_level( existent_path: MagicMock, mock_open: MagicMock, - settings_repository: SecureliRepository, + settings_repository: repo_settings.SecureliRepository, ): - settings_file = SecureliFile() + settings_file = repo_settings.SecureliFile() settings_repository.save(settings_file) mock_open.assert_called_once()