diff --git a/secureli/actions/action.py b/secureli/actions/action.py index b5634f4c..e4c7128b 100644 --- a/secureli/actions/action.py +++ b/secureli/actions/action.py @@ -47,16 +47,24 @@ class Action(ABC): def __init__(self, action_deps: ActionDependencies): self.action_deps = action_deps + def get_secureli_config(self, reset: bool) -> secureli_config.SecureliConfig: + return ( + secureli_config.SecureliConfig() + if reset + else self.action_deps.secureli_config.load() + ) + def verify_install( - self, folder_path: Path, reset: bool, always_yes: bool + self, folder_path: Path, reset: bool, always_yes: bool, files: list[Path] ) -> VerifyResult: """ Installs, upgrades or verifies the current seCureLI installation :param folder_path: The folder path to initialize the repo for :param reset: If true, disregard existing configuration and start fresh :param always_yes: Assume "Yes" to all prompts + :param files: A List of files to scope the install to. This allows language + detection to run on only a selected list of files when scanning the repo. """ - if ( self.action_deps.secureli_config.verify() == secureli_config.VerifyConfigOutcome.OUT_OF_DATE @@ -84,15 +92,11 @@ def verify_install( ) return update_result - config = ( - secureli_config.SecureliConfig() - if reset - else self.action_deps.secureli_config.load() - ) + config = self.get_secureli_config(reset=reset) languages = [] try: - languages = self._detect_languages(folder_path) + languages = self._detect_languages(folder_path, files) except (ValueError, language.LanguageNotSupportedError) as e: if config.languages and config.version_installed: self.action_deps.echo.warning( @@ -296,14 +300,14 @@ def _run_post_install_scan( f"{format_sentence_list(config.languages)} does not support secrets detection, skipping" ) - def _detect_languages(self, folder_path: Path) -> list[str]: + def _detect_languages(self, folder_path: Path, files: list[Path]) -> list[str]: """ Detects programming languages present in the repository :param folder_path: The folder path to initialize the repo for :return: A list of all languages found in the repository """ - analyze_result = self.action_deps.language_analyzer.analyze(folder_path) + analyze_result = self.action_deps.language_analyzer.analyze(folder_path, files) if analyze_result.skipped_files: self.action_deps.echo.warning( diff --git a/secureli/actions/initializer.py b/secureli/actions/initializer.py index 8d7cccab..a07b8bb3 100644 --- a/secureli/actions/initializer.py +++ b/secureli/actions/initializer.py @@ -27,7 +27,7 @@ def initialize_repo( :param reset: If true, disregard existing configuration and start fresh :param always_yes: Assume "Yes" to all prompts """ - verify_result = self.verify_install(folder_path, reset, always_yes) + verify_result = self.verify_install(folder_path, reset, always_yes, files=None) if verify_result.outcome in ScanAction.halting_outcomes: self.logging.failure(LogAction.init, verify_result.outcome) else: diff --git a/secureli/actions/scan.py b/secureli/actions/scan.py index 21fb0f39..f12fd66e 100644 --- a/secureli/actions/scan.py +++ b/secureli/actions/scan.py @@ -3,9 +3,11 @@ from pathlib import Path from time import time from typing import Optional +from git import Repo from secureli.modules.shared.abstractions.echo import EchoAbstraction from secureli.actions import action +from secureli.modules.shared.abstractions.repo import GitRepo from secureli.modules.shared.models.exit_codes import ExitCode from secureli.modules.shared.models.install import VerifyOutcome, VerifyResult from secureli.modules.shared.models.logging import LogAction @@ -37,12 +39,14 @@ def __init__( logging: LoggingService, hooks_scanner: HooksScannerService, pii_scanner: PiiScannerService, + git_repo: GitRepo, ): super().__init__(action_deps) self.hooks_scanner = hooks_scanner self.pii_scanner = pii_scanner self.echo = echo self.logging = logging + self.git_repo = git_repo def _check_secureli_hook_updates(self, folder_path: Path) -> VerifyResult: """ @@ -98,6 +102,24 @@ def publish_results( else: self.logging.failure(LogAction.publish, result.result_message) + def get_commited_files(self, scan_mode: ScanMode) -> list[Path]: + """ + Attempts to build a list of commited files for use in language detection if + the user is scanning staged files for an existing installation + :param scan_mode: Determines which files are scanned in the repo (i.e. staged only or all) + :returns: a list of Path objects for the commited files + """ + config = self.get_secureli_config(reset=False) + installed = bool(config.languages and config.version_installed) + + if not installed or scan_mode != ScanMode.STAGED_ONLY: + return None + try: + committed_files = self.git_repo.get_commit_diff() + return [Path(file) for file in committed_files] + except: + return None + def scan_repo( self, folder_path: Path, @@ -117,7 +139,16 @@ def scan_repo( :param specific_test: If set, limits scanning to the single pre-commit hook. Otherwise, scans with all hooks. """ - verify_result = self.verify_install(folder_path, False, always_yes) + + scan_files = [Path(file) for file in files or []] or self.get_commited_files( + scan_mode + ) + verify_result = self.verify_install( + folder_path, + False, + always_yes, + scan_files, + ) # Check if pre-commit hooks are up-to-date secureli_config = self.action_deps.secureli_config.load() diff --git a/secureli/container.py b/secureli/container.py index afc24de3..b1d548c8 100644 --- a/secureli/container.py +++ b/secureli/container.py @@ -8,6 +8,7 @@ from secureli.actions.scan import ScanAction from secureli.actions.build import BuildAction from secureli.actions.update import UpdateAction +from secureli.modules.shared.abstractions.repo import GitRepo from secureli.repositories.repo_files import RepoFilesRepository from secureli.repositories.secureli_config import SecureliConfigRepository from secureli.repositories.repo_settings import SecureliRepository @@ -83,6 +84,9 @@ class Container(containers.DeclarativeContainer): echo=echo, ) + """Wraps the execution and management of git commands""" + git_repo = providers.Factory(GitRepo) + # Services """Analyzes a set of files to try to determine the most common languages""" @@ -180,6 +184,7 @@ class Container(containers.DeclarativeContainer): logging=logging_service, hooks_scanner=hooks_scanner_service, pii_scanner=pii_scanner_service, + git_repo=git_repo, ) """Update Action, representing what happens when the update command is invoked""" diff --git a/secureli/modules/language_analyzer/language_analyzer.py b/secureli/modules/language_analyzer/language_analyzer.py index 81c0dc69..49c5b110 100644 --- a/secureli/modules/language_analyzer/language_analyzer.py +++ b/secureli/modules/language_analyzer/language_analyzer.py @@ -20,7 +20,7 @@ def __init__( self.repo_files = repo_files self.lexer_guesser = lexer_guesser - def analyze(self, folder_path: Path) -> AnalyzeResult: + def analyze(self, folder_path: Path, files: list[Path]) -> AnalyzeResult: """ Analyzes the folder structure and lists languages found :param folder_path: The path to the repository to analyze @@ -29,7 +29,7 @@ def analyze(self, folder_path: Path) -> AnalyzeResult: 40% of the repo is JavaScript, the result will be a dictionary containing keys "Python" and "JavaScript" with values 0.6 and 0.4 respectively """ - file_paths = self.repo_files.list_repo_files(folder_path) + file_paths = files if files else self.repo_files.list_repo_files(folder_path) results = defaultdict(int) skipped_files = [] diff --git a/secureli/modules/shared/abstractions/repo.py b/secureli/modules/shared/abstractions/repo.py new file mode 100644 index 00000000..6f548447 --- /dev/null +++ b/secureli/modules/shared/abstractions/repo.py @@ -0,0 +1,21 @@ +from abc import ABC, abstractmethod +import git + + +class RepoAbstraction(ABC): + """ + Abstracts the configuring and execution of git repo features. + """ + + @abstractmethod + def get_commit_diff(self) -> list[str]: + pass + + +class GitRepo(RepoAbstraction): + """ + Implementation and wrapper around git repo features + """ + + def get_commit_diff(self) -> list[str]: + return git.Repo().head.commit.diff() diff --git a/tests/actions/test_action.py b/tests/actions/test_action.py index a2fd245f..c9c6faf1 100644 --- a/tests/actions/test_action.py +++ b/tests/actions/test_action.py @@ -63,7 +63,7 @@ def test_that_initialize_repo_raises_value_error_without_any_supported_languages language_proportions={}, skipped_files=[] ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) mock_echo.error.assert_called_with( "seCureLI could not be installed due to an error: No supported languages found in current repository" @@ -83,7 +83,7 @@ def test_that_initialize_repo_install_flow_selects_both_languages( skipped_files=[], ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) mock_echo.print.assert_called_with( "seCureLI has been installed successfully for the following language(s): RadLang and CoolLang.\n", @@ -105,7 +105,7 @@ def test_that_initialize_repo_install_flow_performs_security_analysis( skipped_files=[], ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) mock_hooks_scanner.scan_repo.assert_called_once() @@ -118,7 +118,7 @@ def test_that_initialize_repo_install_flow_displays_security_analysis_results( output="Detect secrets...Failed", failures=[ScanFailure(repo="repo", id="id", file="file")], ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) action_deps.echo.print.assert_any_call("Detect secrets...Failed") @@ -140,7 +140,7 @@ def test_that_initialize_repo_install_flow_skips_security_analysis_if_unavailabl version="abc123", security_hook_id=None ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) mock_hooks_scanner.scan_repo.assert_not_called() @@ -171,7 +171,7 @@ def test_that_initialize_repo_install_flow_warns_about_skipped_files( successful=True, backup_hook_path=None ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) assert ( mock_echo.warning.call_count == 3 @@ -184,7 +184,7 @@ def test_that_initialize_repo_can_be_canceled( ): mock_echo.confirm.return_value = False - action.verify_install(test_folder_path, reset=True, always_yes=False) + action.verify_install(test_folder_path, reset=True, always_yes=False, files=None) mock_echo.error.assert_called_with("User canceled install process") @@ -205,7 +205,7 @@ def test_that_initialize_repo_selects_previously_selected_language( ) mock_language_support.version_for_language.return_value = "abc123" - action.verify_install(test_folder_path, reset=False, always_yes=True) + action.verify_install(test_folder_path, reset=False, always_yes=True, files=None) mock_echo.print.assert_called_with( "seCureLI is installed and up-to-date for the following language(s): PreviousLang" @@ -223,7 +223,7 @@ def test_that_initialize_repo_prompts_to_upgrade_config_if_old_schema( mock_language_support.version_for_language.return_value = "xyz987" mock_echo.confirm.return_value = False - action.verify_install(test_folder_path, reset=False, always_yes=False) + action.verify_install(test_folder_path, reset=False, always_yes=False, files=None) mock_echo.error.assert_called_with("seCureLI could not be verified.") @@ -250,7 +250,9 @@ def test_that_initialize_repo_updates_repo_config_if_old_schema( mock_language_support.version_for_language.return_value = "abc123" - result = action.verify_install(test_folder_path, reset=False, always_yes=True) + result = action.verify_install( + test_folder_path, reset=False, always_yes=True, files=None + ) assert result.outcome == VerifyOutcome.UP_TO_DATE @@ -265,7 +267,7 @@ def test_that_initialize_repo_reports_errors_when_schema_update_fails( mock_secureli_config.update.side_effect = Exception - action.verify_install(test_folder_path, reset=False, always_yes=True) + action.verify_install(test_folder_path, reset=False, always_yes=True, files=None) mock_echo.error.assert_called_with("seCureLI could not be verified.") @@ -280,7 +282,7 @@ def test_that_initialize_repo_is_aborted_by_the_user_if_the_process_is_canceled( mock_echo.confirm.return_value = False mock_secureli_config.load.return_value = SecureliConfig() # fresh config - action.verify_install(test_folder_path, reset=False, always_yes=False) + action.verify_install(test_folder_path, reset=False, always_yes=False, files=None) mock_echo.error.assert_called_with("User canceled install process") @@ -300,7 +302,9 @@ def test_that_initialize_repo_returns_up_to_date_if_the_process_is_canceled_on_e language_proportions={"RadLang": 0.5, "CoolLang": 0.5}, skipped_files=[] ) - result = action.verify_install(test_folder_path, reset=False, always_yes=False) + result = action.verify_install( + test_folder_path, reset=False, always_yes=False, files=None + ) assert result.outcome == VerifyOutcome.UP_TO_DATE @@ -322,7 +326,7 @@ def test_that_initialize_repo_prints_warnings_for_failed_linter_config_writes( successful=True, backup_hook_path=None ) - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) mock_echo.warning.assert_called_once_with(config_write_error) @@ -341,7 +345,7 @@ def test_that_verify_install_returns_failed_result_on_new_install_language_not_s ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=False + test_folder_path, reset=False, always_yes=False, files=None ) assert verify_result.outcome == VerifyOutcome.INSTALL_FAILED @@ -361,7 +365,7 @@ def test_that_verify_install_returns_up_to_date_result_on_existing_install_langu ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=False + test_folder_path, reset=False, always_yes=False, files=None ) assert verify_result.outcome == VerifyOutcome.UP_TO_DATE @@ -381,7 +385,7 @@ def test_that_verify_install_returns_up_to_date_result_on_existing_install_no_ne ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=False + test_folder_path, reset=False, always_yes=False, files=None ) assert verify_result.outcome == VerifyOutcome.UP_TO_DATE @@ -401,7 +405,7 @@ def test_that_verify_install_returns_success_result_newly_detected_language_inst ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=True + test_folder_path, reset=False, always_yes=True, files=None ) assert verify_result.outcome == VerifyOutcome.INSTALL_SUCCEEDED @@ -420,7 +424,7 @@ def test_that_verify_install_returns_failure_result_without_re_commit_config_fil "ERROR" ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=True + test_folder_path, reset=False, always_yes=True, files=None ) mock_echo.error.assert_called_once_with( "seCureLI pre-commit-config.yaml could not be updated." @@ -438,7 +442,7 @@ def test_that_verify_install_continues_after_pre_commit_config_file_moved( test_folder_path / ".secureli" / ".pre-commit-config.yaml" ) verify_result = action.verify_install( - test_folder_path, reset=False, always_yes=True + test_folder_path, reset=False, always_yes=True, files=None ) assert verify_result.outcome == VerifyOutcome.INSTALL_SUCCEEDED @@ -512,7 +516,7 @@ def test_that_initialize_repo_install_flow_warns_about_overwriting_pre_commit_fi mock_updater.pre_commit.install.return_value = install_result - action.verify_install(test_folder_path, reset=True, always_yes=True) + action.verify_install(test_folder_path, reset=True, always_yes=True, files=None) mock_echo.warning.assert_called_once_with( ( diff --git a/tests/actions/test_scan_action.py b/tests/actions/test_scan_action.py index 826307b6..8724ada8 100644 --- a/tests/actions/test_scan_action.py +++ b/tests/actions/test_scan_action.py @@ -69,6 +69,11 @@ def mock_updater() -> MagicMock: return mock_updater +@pytest.fixture() +def mock_git_repo() -> MagicMock: + return MagicMock() + + @pytest.fixture() def mock_get_time_near_epoch(mocker: MockerFixture) -> MagicMock: return mocker.patch( @@ -126,6 +131,7 @@ def scan_action( action_deps: ActionDependencies, mock_logging_service: MagicMock, mock_pii_scanner: MagicMock, + mock_git_repo: MagicMock, ) -> ScanAction: return ScanAction( action_deps=action_deps, @@ -133,6 +139,7 @@ def scan_action( logging=mock_logging_service, hooks_scanner=action_deps.hooks_scanner, pii_scanner=mock_pii_scanner, + git_repo=mock_git_repo, ) @@ -389,3 +396,116 @@ def test_publish_results_on_fail_and_action_not_successful( mock_post_log.assert_called_once_with("log_str", Settings()) scan_action.logging.failure.assert_called_once_with(LogAction.publish, "Failure") + + +def test_verify_install_is_called_with_commted_files( + scan_action: ScanAction, + mock_git_repo: MagicMock, + mock_secureli_config: MagicMock, + mock_language_analyzer: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed=1 + ) + + mock_files = ["file1.py", "file2.py"] + + mock_git_repo.get_commit_diff.return_value = mock_files + scan_action.scan_repo( + folder_path=Path(""), + scan_mode=ScanMode.STAGED_ONLY, + always_yes=True, + publish_results_condition=PublishResultsOption.NEVER, + specific_test=None, + files=None, + ) + + mock_language_analyzer.analyze.assert_called_once_with( + Path("."), [Path(file) for file in mock_files] + ) + + +def test_verify_install_is_called_with_user_specified_files( + scan_action: ScanAction, + mock_git_repo: MagicMock, + mock_secureli_config: MagicMock, + mock_language_analyzer: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed=1 + ) + + mock_files = ["file1.py", "file2.py"] + + mock_git_repo.get_commit_diff.return_value = None + scan_action.scan_repo( + folder_path=Path(""), + scan_mode=ScanMode.STAGED_ONLY, + always_yes=True, + publish_results_condition=PublishResultsOption.NEVER, + specific_test=None, + files=mock_files, + ) + + mock_language_analyzer.analyze.assert_called_once_with( + Path("."), [Path(file) for file in mock_files] + ) + + +def test_verify_install_is_called_with_no_specified_files( + scan_action: ScanAction, + mock_git_repo: MagicMock, + mock_secureli_config: MagicMock, + mock_language_analyzer: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed=1 + ) + + mock_git_repo.get_commit_diff.return_value = None + scan_action.scan_repo( + folder_path=Path(""), + scan_mode=ScanMode.STAGED_ONLY, + always_yes=True, + publish_results_condition=PublishResultsOption.NEVER, + specific_test=None, + files=None, + ) + + mock_language_analyzer.analyze.assert_called_once_with(Path("."), None) + + +def test_get_commited_files_returns_commit_diff( + scan_action: ScanAction, + mock_git_repo: MagicMock, + mock_secureli_config: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed=1 + ) + mock_files = [Path("file1.py"), Path("file2.py")] + mock_git_repo.get_commit_diff.return_value = mock_files + result = scan_action.get_commited_files(scan_mode=ScanMode.STAGED_ONLY) + assert result == mock_files + + +def test_get_commited_files_returns_none_when_not_installed( + scan_action: ScanAction, + mock_secureli_config: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=[], version_installed=None + ) + result = scan_action.get_commited_files(scan_mode=ScanMode.STAGED_ONLY) + assert result is None + + +def test_get_commited_files_returns_when_scan_mode_is_not_staged_only( + scan_action: ScanAction, + mock_secureli_config: MagicMock, +): + mock_secureli_config.load.return_value = SecureliConfig( + languages=["RadLang"], version_installed=1 + ) + result = scan_action.get_commited_files(scan_mode=ScanMode.ALL_FILES) + assert result is None diff --git a/tests/modules/language_analyzer/test_language_analyzer.py b/tests/modules/language_analyzer/test_language_analyzer.py index 8a094e1e..e5ea9680 100644 --- a/tests/modules/language_analyzer/test_language_analyzer.py +++ b/tests/modules/language_analyzer/test_language_analyzer.py @@ -87,7 +87,9 @@ def test_that_language_analyzer_removes_unsupported_languages( language_analyzer_bad_lang: language_analyzer.LanguageAnalyzerService, folder_path: MagicMock, ): - percentages_per_language = language_analyzer_bad_lang.analyze(folder_path) + percentages_per_language = language_analyzer_bad_lang.analyze( + folder_path, files=None + ) assert "BadLang" not in percentages_per_language @@ -96,7 +98,7 @@ def test_that_language_analyzer_includes_python( language_analyzer_python: language_analyzer.LanguageAnalyzerService, folder_path: MagicMock, ): - analyze_result = language_analyzer_python.analyze(folder_path) + analyze_result = language_analyzer_python.analyze(folder_path, files=None) assert "Python" in analyze_result.language_proportions assert analyze_result.language_proportions["Python"] == 1.0 @@ -106,6 +108,6 @@ def test_that_language_analyzer_displays_warnings( language_analyzer_with_warnings: language_analyzer.LanguageAnalyzerService, folder_path: MagicMock, ): - analyze_result = language_analyzer_with_warnings.analyze(folder_path) + analyze_result = language_analyzer_with_warnings.analyze(folder_path, files=None) assert len(analyze_result.skipped_files) == 3 diff --git a/tests/modules/shared/abstractions/test_repo.py b/tests/modules/shared/abstractions/test_repo.py new file mode 100644 index 00000000..5a99462d --- /dev/null +++ b/tests/modules/shared/abstractions/test_repo.py @@ -0,0 +1,25 @@ +from pathlib import Path +from unittest.mock import MagicMock +import pytest +from pytest_mock import MockerFixture + +from secureli.modules.shared.abstractions.repo import GitRepo + + +@pytest.fixture() +def git_repo() -> GitRepo: + return GitRepo() + + +@pytest.fixture() +def mock_repo(mocker: MockerFixture) -> MagicMock: + return mocker.patch("git.Repo", MagicMock()) + + +def test_that_get_commit_diff_returns_file_diff( + git_repo: GitRepo, mock_repo: MagicMock +): + mock_files = [Path("test_file.py"), Path("test_file2.py")] + mock_repo.return_value.head.commit.diff.return_value = mock_files + result = git_repo.get_commit_diff() + assert result is mock_files