From c63860f45564502afc48ad8afe5113f614127b78 Mon Sep 17 00:00:00 2001 From: kevin-orlando <58826693+kevin-orlando@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:31:26 -0500 Subject: [PATCH] feat: 154 Prompt User to Install Code Linters for each Detected Language (#332) closes #154 **Overview** Adds functionality to prompt the user to determine if linter based pre-commit hooks should be added to the code repository. The user will be prompted for each detected language during `init`. An example messages will be `Add lint pre-commit(s) for JavaScript? [Y/n]` adding the `--yes` option will bypass the prompting to install linter pre-commit hooks and will automatically add them. **Technical Approach** This pr includes reorganizing the pre-commit templates into a new folder/file structure. Templates will be located under `resources/pre-commit` and will be split into separate folders and files based on if they are linter hooks or not. The user responses and code will determine whether or not the hooks should be combined and saved to the user's repository. Splitting these files out ensures a simple way of knowing which hooks are linters. **Testing** 1. run `secureli init` 2. follow flow to add/ignore linters for each detected language 3. Verify pre-commit linters are added or not added to pre-commit.yaml depending on prompt response Regression testing: Testing `scan` and `update` to ensure both are working as normal. --- .coveragerc | 1 + CONTRIBUTING.md | 1 + secureli/actions/action.py | 39 +++++++- secureli/repositories/secureli_config.py | 2 + secureli/resources/files/base-pre-commit.yaml | 27 ------ .../resources/files/csharp-pre-commit.yaml | 14 --- secureli/resources/files/java-pre-commit.yaml | 5 -- .../files/javascript-pre-commit.yaml | 21 ----- .../pre-commit/base/base-pre-commit.yaml | 27 ++++++ .../pre-commit/base/csharp-pre-commit.yaml | 1 + .../files/pre-commit/base/go-pre-commit.yaml | 1 + .../pre-commit/base/java-pre-commit.yaml | 1 + .../base/javascript-pre-commit.yaml | 1 + .../pre-commit/base/python-pre-commit.yaml | 10 +++ .../pre-commit/base/swift-pre-commit.yaml | 6 ++ .../pre-commit/base/terraform-pre-commit.yaml | 5 ++ .../base/typescript-pre-commit.yaml | 1 + .../pre-commit/lint/base-pre-commit.yaml | 1 + .../pre-commit/lint/csharp-pre-commit.yaml | 14 +++ .../{ => pre-commit/lint}/go-pre-commit.yaml | 2 +- .../pre-commit/lint/java-pre-commit.yaml | 5 ++ .../lint/javascript-pre-commit.yaml | 21 +++++ .../pre-commit/lint/python-pre-commit.yaml | 5 ++ .../pre-commit/lint/swift-pre-commit.yaml | 5 ++ .../pre-commit/lint/terraform-pre-commit.yaml | 5 ++ .../lint/typescript-pre-commit.yaml | 23 +++++ .../secrets_detecting_repos.yaml | 0 .../resources/files/python-pre-commit.yaml | 14 --- .../resources/files/swift-pre-commit.yaml | 10 --- .../resources/files/terraform-pre-commit.yaml | 9 -- .../files/typescript-pre-commit.yaml | 23 ----- secureli/services/language_config.py | 45 ++++++++-- secureli/services/language_support.py | 35 ++++---- tests/actions/test_action.py | 61 +++++++++++++ tests/application/test_main.py | 9 ++ tests/services/test_language_config.py | 89 +++++++++++++++---- tests/services/test_language_support.py | 58 +++++++++++- 37 files changed, 428 insertions(+), 169 deletions(-) delete mode 100644 secureli/resources/files/base-pre-commit.yaml delete mode 100644 secureli/resources/files/csharp-pre-commit.yaml delete mode 100644 secureli/resources/files/java-pre-commit.yaml delete mode 100644 secureli/resources/files/javascript-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/base/base-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/base/csharp-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/base/go-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/base/java-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/base/javascript-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/base/python-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/base/swift-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/base/terraform-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/base/typescript-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/lint/base-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/lint/csharp-pre-commit.yaml rename secureli/resources/files/{ => pre-commit/lint}/go-pre-commit.yaml (92%) create mode 100644 secureli/resources/files/pre-commit/lint/java-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/lint/javascript-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/lint/python-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/lint/swift-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/lint/terraform-pre-commit.yaml create mode 100644 secureli/resources/files/pre-commit/lint/typescript-pre-commit.yaml rename secureli/resources/files/{ => pre-commit}/secrets_detecting_repos.yaml (100%) delete mode 100644 secureli/resources/files/python-pre-commit.yaml delete mode 100644 secureli/resources/files/swift-pre-commit.yaml delete mode 100644 secureli/resources/files/terraform-pre-commit.yaml delete mode 100644 secureli/resources/files/typescript-pre-commit.yaml diff --git a/.coveragerc b/.coveragerc index d0f7789b..0908e672 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,6 +4,7 @@ source = secureli [report] fail_under = 90 +show_missing = true omit = tests/* */__init__.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a1d92cd..cc4e86be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -300,3 +300,4 @@ A special thanks to everyone that has contributed to seCureLI so far: - Jeff Schumacher - Caleb Tonn - Josh Werner +- Kevin Orlando diff --git a/secureli/actions/action.py b/secureli/actions/action.py index fd1ddd84..2b4a4d8e 100644 --- a/secureli/actions/action.py +++ b/secureli/actions/action.py @@ -148,7 +148,19 @@ def _install_secureli(self, folder_path: Path, always_yes: bool) -> VerifyResult languages = list(analyze_result.language_proportions.keys()) self.action_deps.echo.print(f"Overall Detected Languages: {languages}") - metadata = self.action_deps.language_support.apply_support(languages) + lint_languages = self._prompt_get_lint_config_languages( + languages, always_yes + ) + + language_config_result = ( + self.action_deps.language_support._build_pre_commit_config( + languages, lint_languages + ) + ) + + metadata = self.action_deps.language_support.apply_support( + languages, language_config_result + ) except (ValueError, LanguageNotSupportedError, InstallFailedError) as e: self.action_deps.echo.error( @@ -160,6 +172,7 @@ def _install_secureli(self, folder_path: Path, always_yes: bool) -> VerifyResult config = SecureliConfig( languages=languages, + lint_languages=lint_languages, version_installed=metadata.version, ) self.action_deps.secureli_config.save(config) @@ -193,6 +206,30 @@ def _install_secureli(self, folder_path: Path, always_yes: bool) -> VerifyResult analyze_result=analyze_result, ) + def _prompt_get_lint_config_languages( + self, languages: list[str], always_yes: bool + ) -> list[str]: + """ + Prompts user to add lint pre-commit hooks for each detected language + :param languages: list of detected languages + :param always_yes: Assume "Yes" to all prompts + :return: set of filtered languages to add lint pre-commit hooks for + """ + if always_yes: + return [*languages] + + lint_languages: list[str] = [] + + for language in languages: + add_linter = self.action_deps.echo.confirm( + f"Add lint pre-commit hook(s) for {language}?", default_response=True + ) + + if add_linter: + lint_languages.append(language) + + return lint_languages + def _update_secureli(self, always_yes: bool): """ Prompts the user to update to the latest secureli install. diff --git a/secureli/repositories/secureli_config.py b/secureli/repositories/secureli_config.py index cc29cef1..fb721223 100644 --- a/secureli/repositories/secureli_config.py +++ b/secureli/repositories/secureli_config.py @@ -10,6 +10,7 @@ class SecureliConfig(BaseModel): languages: Optional[list[str]] + lint_languages: Optional[list[str]] version_installed: Optional[str] @@ -96,6 +97,7 @@ def update(self) -> SecureliConfig: return SecureliConfig( languages=[old_config.overall_language], + lint_languages=[old_config.overall_language], version_installed=old_config.version_installed, ) diff --git a/secureli/resources/files/base-pre-commit.yaml b/secureli/resources/files/base-pre-commit.yaml deleted file mode 100644 index 1dd622d8..00000000 --- a/secureli/resources/files/base-pre-commit.yaml +++ /dev/null @@ -1,27 +0,0 @@ -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 - hooks: - - id: check-added-large-files - - id: check-ast - - id: check-docstring-first - - id: check-executables-have-shebangs - - id: check-shebang-scripts-are-executable - - id: check-merge-conflict - - id: check-toml - - id: check-json - - id: check-xml - - id: check-yaml - - id: debug-statements - - id: detect-aws-credentials - args: [--allow-missing-credentials] - - id: detect-private-key - - id: name-tests-test - args: [--pytest-test-first] - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml -- repo: https://github.com/Yelp/detect-secrets - rev: v1.4.0 - hooks: - - id: detect-secrets diff --git a/secureli/resources/files/csharp-pre-commit.yaml b/secureli/resources/files/csharp-pre-commit.yaml deleted file mode 100644 index f497f19a..00000000 --- a/secureli/resources/files/csharp-pre-commit.yaml +++ /dev/null @@ -1,14 +0,0 @@ -repos: -- repo: local - hooks: - # Note: The dotnet format pre-commit setup combines poorly to be tightly coupled with - # a pre-release version of .net that is old and no one has installed. dotnet format has - # since become a part of .net! So we can use dotnet format already installed on your - # simply. This runs the risk that different folks will run different versions, but - # this is better than nothing. - # see https://github.com/dotnet/format/issues/1350 and the resolution PR at the bottom. - - id: dotnet-format - name: dotnet-format - language: system - entry: dotnet format --include - types: ["c#"] diff --git a/secureli/resources/files/java-pre-commit.yaml b/secureli/resources/files/java-pre-commit.yaml deleted file mode 100644 index 04d2a2d5..00000000 --- a/secureli/resources/files/java-pre-commit.yaml +++ /dev/null @@ -1,5 +0,0 @@ -repos: -- repo: https://github.com/slalombuild/pre-commit-mirror-checkstyle - rev: v0.1.1 - hooks: - - id: checkstyle-java diff --git a/secureli/resources/files/javascript-pre-commit.yaml b/secureli/resources/files/javascript-pre-commit.yaml deleted file mode 100644 index bffbf27b..00000000 --- a/secureli/resources/files/javascript-pre-commit.yaml +++ /dev/null @@ -1,21 +0,0 @@ -repos: -- repo: https://github.com/pre-commit/mirrors-eslint - rev: 'v8.42.0' - hooks: - - id: eslint - files: \.[j]sx?$ # *.js and *.jsx - types: [file] - args: ['--config', '.secureli/javascript.eslintrc.yaml', '--fix'] - additional_dependencies: - - eslint@8.42.0 - - eslint-config-google@0.7.1 - - eslint-plugin-prettier@4.2.1 -- repo: https://github.com/pre-commit/mirrors-prettier - rev: 'v2.7.1' - hooks: - - id: prettier - args: - - --single-quote - - --trailing-comma - - all - types_or: [css, javascript] diff --git a/secureli/resources/files/pre-commit/base/base-pre-commit.yaml b/secureli/resources/files/pre-commit/base/base-pre-commit.yaml new file mode 100644 index 00000000..b86a802d --- /dev/null +++ b/secureli/resources/files/pre-commit/base/base-pre-commit.yaml @@ -0,0 +1,27 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: check-merge-conflict + - id: check-toml + - id: check-json + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: detect-aws-credentials + args: [--allow-missing-credentials] + - id: detect-private-key + - id: name-tests-test + args: [--pytest-test-first] + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - repo: https://github.com/Yelp/detect-secrets + rev: v1.4.0 + hooks: + - id: detect-secrets diff --git a/secureli/resources/files/pre-commit/base/csharp-pre-commit.yaml b/secureli/resources/files/pre-commit/base/csharp-pre-commit.yaml new file mode 100644 index 00000000..18f04637 --- /dev/null +++ b/secureli/resources/files/pre-commit/base/csharp-pre-commit.yaml @@ -0,0 +1 @@ +repos: diff --git a/secureli/resources/files/pre-commit/base/go-pre-commit.yaml b/secureli/resources/files/pre-commit/base/go-pre-commit.yaml new file mode 100644 index 00000000..18f04637 --- /dev/null +++ b/secureli/resources/files/pre-commit/base/go-pre-commit.yaml @@ -0,0 +1 @@ +repos: diff --git a/secureli/resources/files/pre-commit/base/java-pre-commit.yaml b/secureli/resources/files/pre-commit/base/java-pre-commit.yaml new file mode 100644 index 00000000..18f04637 --- /dev/null +++ b/secureli/resources/files/pre-commit/base/java-pre-commit.yaml @@ -0,0 +1 @@ +repos: diff --git a/secureli/resources/files/pre-commit/base/javascript-pre-commit.yaml b/secureli/resources/files/pre-commit/base/javascript-pre-commit.yaml new file mode 100644 index 00000000..18f04637 --- /dev/null +++ b/secureli/resources/files/pre-commit/base/javascript-pre-commit.yaml @@ -0,0 +1 @@ +repos: diff --git a/secureli/resources/files/pre-commit/base/python-pre-commit.yaml b/secureli/resources/files/pre-commit/base/python-pre-commit.yaml new file mode 100644 index 00000000..2fef99f2 --- /dev/null +++ b/secureli/resources/files/pre-commit/base/python-pre-commit.yaml @@ -0,0 +1,10 @@ +repos: + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.9.0 + hooks: + - id: python-use-type-annotations + - repo: https://github.com/PyCQA/bandit + rev: 1.7.4 + hooks: + - id: bandit + args: ["--exclude", "tests/", "--severity-level", "medium"] diff --git a/secureli/resources/files/pre-commit/base/swift-pre-commit.yaml b/secureli/resources/files/pre-commit/base/swift-pre-commit.yaml new file mode 100644 index 00000000..bd7f985e --- /dev/null +++ b/secureli/resources/files/pre-commit/base/swift-pre-commit.yaml @@ -0,0 +1,6 @@ +repos: + - repo: https://github.com/Yelp/detect-secrets + rev: v1.4.0 + hooks: + - id: detect-secrets + exclude: .xcscheme$ diff --git a/secureli/resources/files/pre-commit/base/terraform-pre-commit.yaml b/secureli/resources/files/pre-commit/base/terraform-pre-commit.yaml new file mode 100644 index 00000000..3dbfe107 --- /dev/null +++ b/secureli/resources/files/pre-commit/base/terraform-pre-commit.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/Yelp/detect-secrets + rev: v1.4.0 + hooks: + - id: detect-secrets diff --git a/secureli/resources/files/pre-commit/base/typescript-pre-commit.yaml b/secureli/resources/files/pre-commit/base/typescript-pre-commit.yaml new file mode 100644 index 00000000..18f04637 --- /dev/null +++ b/secureli/resources/files/pre-commit/base/typescript-pre-commit.yaml @@ -0,0 +1 @@ +repos: diff --git a/secureli/resources/files/pre-commit/lint/base-pre-commit.yaml b/secureli/resources/files/pre-commit/lint/base-pre-commit.yaml new file mode 100644 index 00000000..18f04637 --- /dev/null +++ b/secureli/resources/files/pre-commit/lint/base-pre-commit.yaml @@ -0,0 +1 @@ +repos: diff --git a/secureli/resources/files/pre-commit/lint/csharp-pre-commit.yaml b/secureli/resources/files/pre-commit/lint/csharp-pre-commit.yaml new file mode 100644 index 00000000..47af8498 --- /dev/null +++ b/secureli/resources/files/pre-commit/lint/csharp-pre-commit.yaml @@ -0,0 +1,14 @@ +repos: + - repo: local + hooks: + # Note: The dotnet format pre-commit setup combines poorly to be tightly coupled with + # a pre-release version of .net that is old and no one has installed. dotnet format has + # since become a part of .net! So we can use dotnet format already installed on your + # simply. This runs the risk that different folks will run different versions, but + # this is better than nothing. + # see https://github.com/dotnet/format/issues/1350 and the resolution PR at the bottom. + - id: dotnet-format + name: dotnet-format + language: system + entry: dotnet format --include + types: ["c#"] diff --git a/secureli/resources/files/go-pre-commit.yaml b/secureli/resources/files/pre-commit/lint/go-pre-commit.yaml similarity index 92% rename from secureli/resources/files/go-pre-commit.yaml rename to secureli/resources/files/pre-commit/lint/go-pre-commit.yaml index bd54995b..5a5b46b2 100644 --- a/secureli/resources/files/go-pre-commit.yaml +++ b/secureli/resources/files/pre-commit/lint/go-pre-commit.yaml @@ -1,4 +1,4 @@ - repos: +repos: - repo: https://github.com/golangci/golangci-lint rev: v1.53.3 hooks: diff --git a/secureli/resources/files/pre-commit/lint/java-pre-commit.yaml b/secureli/resources/files/pre-commit/lint/java-pre-commit.yaml new file mode 100644 index 00000000..b0633f57 --- /dev/null +++ b/secureli/resources/files/pre-commit/lint/java-pre-commit.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/slalombuild/pre-commit-mirror-checkstyle + rev: v0.1.1 + hooks: + - id: checkstyle-java diff --git a/secureli/resources/files/pre-commit/lint/javascript-pre-commit.yaml b/secureli/resources/files/pre-commit/lint/javascript-pre-commit.yaml new file mode 100644 index 00000000..278767f3 --- /dev/null +++ b/secureli/resources/files/pre-commit/lint/javascript-pre-commit.yaml @@ -0,0 +1,21 @@ +repos: + - repo: https://github.com/pre-commit/mirrors-eslint + rev: "v8.42.0" + hooks: + - id: eslint + files: \.[j]sx?$ # *.js and *.jsx + types: [file] + args: ["--config", ".secureli/javascript.eslintrc.yaml", "--fix"] + additional_dependencies: + - eslint@8.42.0 + - eslint-config-google@0.7.1 + - eslint-plugin-prettier@4.2.1 + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v2.7.1" + hooks: + - id: prettier + args: + - --single-quote + - --trailing-comma + - all + types_or: [css, javascript] diff --git a/secureli/resources/files/pre-commit/lint/python-pre-commit.yaml b/secureli/resources/files/pre-commit/lint/python-pre-commit.yaml new file mode 100644 index 00000000..fe30c3b4 --- /dev/null +++ b/secureli/resources/files/pre-commit/lint/python-pre-commit.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black diff --git a/secureli/resources/files/pre-commit/lint/swift-pre-commit.yaml b/secureli/resources/files/pre-commit/lint/swift-pre-commit.yaml new file mode 100644 index 00000000..2be53df1 --- /dev/null +++ b/secureli/resources/files/pre-commit/lint/swift-pre-commit.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/realm/SwiftLint + rev: 0.52.2 + hooks: + - id: swiftlint diff --git a/secureli/resources/files/pre-commit/lint/terraform-pre-commit.yaml b/secureli/resources/files/pre-commit/lint/terraform-pre-commit.yaml new file mode 100644 index 00000000..075364b4 --- /dev/null +++ b/secureli/resources/files/pre-commit/lint/terraform-pre-commit.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.77.0 + hooks: + - id: terraform_tflint diff --git a/secureli/resources/files/pre-commit/lint/typescript-pre-commit.yaml b/secureli/resources/files/pre-commit/lint/typescript-pre-commit.yaml new file mode 100644 index 00000000..6f0691cd --- /dev/null +++ b/secureli/resources/files/pre-commit/lint/typescript-pre-commit.yaml @@ -0,0 +1,23 @@ +repos: + - repo: https://github.com/pre-commit/mirrors-eslint + rev: "v8.42.0" + hooks: + - id: eslint + files: \.[t]sx?$ # *.ts and *.tsx + types: [file] + args: ["--config", ".secureli/typescript.eslintrc.yaml", "--fix"] + additional_dependencies: + - eslint@8.42.0 + - eslint-config-google@0.7.1 + - "@typescript-eslint/eslint-plugin@5.59.11" + - "@typescript-eslint/parser@5.59.11" + - typescript@5.1.3 + - eslint-config-prettier@8.8.0 + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v2.7.1" + hooks: + - id: prettier + args: + - --single-quote + - --trailing-comma + - all diff --git a/secureli/resources/files/secrets_detecting_repos.yaml b/secureli/resources/files/pre-commit/secrets_detecting_repos.yaml similarity index 100% rename from secureli/resources/files/secrets_detecting_repos.yaml rename to secureli/resources/files/pre-commit/secrets_detecting_repos.yaml diff --git a/secureli/resources/files/python-pre-commit.yaml b/secureli/resources/files/python-pre-commit.yaml deleted file mode 100644 index d5681f98..00000000 --- a/secureli/resources/files/python-pre-commit.yaml +++ /dev/null @@ -1,14 +0,0 @@ -repos: -- repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.9.0 - hooks: - - id: python-use-type-annotations -- repo: https://github.com/PyCQA/bandit - rev: 1.7.4 - hooks: - - id: bandit - args: ["--exclude", "tests/", "--severity-level", "medium"] -- repo: https://github.com/psf/black - rev: 22.10.0 - hooks: - - id: black diff --git a/secureli/resources/files/swift-pre-commit.yaml b/secureli/resources/files/swift-pre-commit.yaml deleted file mode 100644 index 0ebbd7fb..00000000 --- a/secureli/resources/files/swift-pre-commit.yaml +++ /dev/null @@ -1,10 +0,0 @@ -repos: -- repo: https://github.com/Yelp/detect-secrets - rev: v1.4.0 - hooks: - - id: detect-secrets - exclude: .xcscheme$ -- repo: https://github.com/realm/SwiftLint - rev: 0.52.2 - hooks: - - id: swiftlint diff --git a/secureli/resources/files/terraform-pre-commit.yaml b/secureli/resources/files/terraform-pre-commit.yaml deleted file mode 100644 index 9c61f3f1..00000000 --- a/secureli/resources/files/terraform-pre-commit.yaml +++ /dev/null @@ -1,9 +0,0 @@ -repos: -- repo: https://github.com/Yelp/detect-secrets - rev: v1.4.0 - hooks: - - id: detect-secrets -- repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.0 - hooks: - - id: terraform_tflint diff --git a/secureli/resources/files/typescript-pre-commit.yaml b/secureli/resources/files/typescript-pre-commit.yaml deleted file mode 100644 index 185bb847..00000000 --- a/secureli/resources/files/typescript-pre-commit.yaml +++ /dev/null @@ -1,23 +0,0 @@ -repos: -- repo: https://github.com/pre-commit/mirrors-eslint - rev: 'v8.42.0' - hooks: - - id: eslint - files: \.[t]sx?$ # *.ts and *.tsx - types: [file] - args: ['--config', '.secureli/typescript.eslintrc.yaml', '--fix'] - additional_dependencies: - - eslint@8.42.0 - - eslint-config-google@0.7.1 - - '@typescript-eslint/eslint-plugin@5.59.11' - - '@typescript-eslint/parser@5.59.11' - - typescript@5.1.3 - - eslint-config-prettier@8.8.0 -- repo: https://github.com/pre-commit/mirrors-prettier - rev: 'v2.7.1' - hooks: - - id: prettier - args: - - --single-quote - - --trailing-comma - - all diff --git a/secureli/services/language_config.py b/secureli/services/language_config.py index 307a8ffc..34c9c011 100644 --- a/secureli/services/language_config.py +++ b/secureli/services/language_config.py @@ -45,19 +45,27 @@ def __init__( self.data_loader = data_loader self.ignored_file_patterns = ignored_file_patterns - def get_language_config(self, language: str) -> LanguagePreCommitResult: + def get_language_config( + self, language: str, include_linter: bool + ) -> LanguagePreCommitResult: """ Calculates a hash of the pre-commit file for the given language to be used as part of the overall installed configuration. :param language: The language specified + :param include_linter: Whether or not linter pre-commit hooks/configs should be included :raises LanguageNotSupportedError if the associated pre-commit file for the language is not found :return: LanguagePreCommitConfig - A configuration model containing the language, config file data, and a versioning hash of the file contents. """ try: - config_data = self._calculate_combined_configuration_data(language) - linter_config_data = self._load_linter_config_file(language) - + config_data = self._calculate_combined_configuration_data( + language, include_linter + ) + linter_config_data = ( + self._load_linter_config_file(language) + if include_linter + else LoadLinterConfigsResult(successful=True, linter_data=list()) + ) version = hash_config(config_data) return LanguagePreCommitResult( language=language, @@ -70,31 +78,50 @@ def get_language_config(self, language: str) -> LanguagePreCommitResult: f"Language '{language}' is currently unsupported" ) - def _calculate_combined_configuration(self, language: str) -> dict: + def _calculate_combined_configuration( + self, language: str, include_linter: bool + ) -> dict: """ Combine elements of our configuration for the specified language along with repo settings like ignored file patterns and pre-commit overrides :param language: The language to load the configuration for as a basis for the combined configuration + :param include_linter: Determines whether or not the lint pre-commit repos + should be added to the configuration result :return: The combined configuration data as a dictionary """ + config = {"repos": []} slugified_language = slugify(language) - config_data = self.data_loader(f"{slugified_language}-pre-commit.yaml") - config = yaml.safe_load(config_data) or {} + config_folder_names = ["base"] + + if include_linter: + config_folder_names.append("lint") + + for folder_name in config_folder_names: + config_data = self.data_loader( + f"pre-commit/{folder_name}/{slugified_language}-pre-commit.yaml" + ) + parsed_config = yaml.safe_load(config_data) or {"repos": None} + repos = parsed_config["repos"] + config["repos"] += repos or [] + if self.ignored_file_patterns: config["exclude"] = combine_patterns(self.ignored_file_patterns) return config - def _calculate_combined_configuration_data(self, language: str) -> str: + def _calculate_combined_configuration_data( + self, language: str, include_linter: bool + ) -> str: """ Combine elements of our configuration for the specified language along with repo settings like ignored file patterns and future overrides :param language: The language to load the configuration for as a basis for the combined configuration + :param include_linter: Whether or not linter pre-commit hooks should be included :return: The combined configuration data as a string """ - config = self._calculate_combined_configuration(language) + config = self._calculate_combined_configuration(language, include_linter) return yaml.dump(config) def _load_linter_config_file(self, language: str) -> LoadLinterConfigsResult: diff --git a/secureli/services/language_support.py b/secureli/services/language_support.py index 3e492a7f..18fb3ed8 100644 --- a/secureli/services/language_support.py +++ b/secureli/services/language_support.py @@ -101,7 +101,9 @@ def __init__( self.language_config = language_config self.data_loader = data_loader - def apply_support(self, languages: list[str]) -> LanguageMetadata: + def apply_support( + self, languages: list[str], language_config_result: BuildConfigResult + ) -> LanguageMetadata: """ Applies Secure Build support for the provided language :param languages: list of languages to provide support for @@ -111,8 +113,6 @@ def apply_support(self, languages: list[str]) -> LanguageMetadata: """ path_to_pre_commit_file = SecureliConfig.FOLDER_PATH / ".pre-commit-config.yaml" - # Raises a LanguageNotSupportedError if language doesn't resolve to a yaml file - language_config_result = self._build_pre_commit_config(languages) if len(language_config_result.linter_configs) > 0: self._write_pre_commit_configs(language_config_result.linter_configs) @@ -136,9 +136,12 @@ def secret_detection_hook_id(self, languages: list[str]) -> Optional[str]: :param languages: list of languages to check support for :return: The hook ID to use for secrets analysis if supported, otherwise None. """ - language_config = self._build_pre_commit_config(languages) + # lint_languages param can be an empty set since we only need secrets detection hooks + language_config = self._build_pre_commit_config(languages, []) config = language_config.config_data - secrets_detecting_repos_data = self.data_loader("secrets_detecting_repos.yaml") + secrets_detecting_repos_data = self.data_loader( + "pre-commit/secrets_detecting_repos.yaml" + ) secrets_detecting_repos = yaml.safe_load(secrets_detecting_repos_data) # Make sure the repos and configuration don't care about case sensitivity @@ -177,7 +180,7 @@ def get_configuration(self, languages: list[str]) -> HookConfiguration: :param languages: list of languages to get config for. :return: A serializable Configuration model """ - config = self._build_pre_commit_config(languages).config_data + config = self._build_pre_commit_config(languages, set(languages)).config_data def create_repo(raw_repo: dict) -> Repo: return Repo( @@ -189,21 +192,25 @@ def create_repo(raw_repo: dict) -> Repo: repos = [create_repo(raw_repo) for raw_repo in config.get("repos", [])] return HookConfiguration(repos=repos) - def _build_pre_commit_config(self, languages: list[str]) -> BuildConfigResult: + def _build_pre_commit_config( + self, languages: list[str], lint_languages: list[str] + ) -> BuildConfigResult: """ Builds the final .pre-commit-config.yaml from all supported repo languages. Also returns any and all linter configuration data. :param languages: list of languages to get calculated configuration for. + :param lint_languages: list of languages to add lint pre-commit hooks for. :return: BuildConfigResult """ config_data = [] - successful_languages = [] + successful_languages: list[str] = [] linter_configs: list[LinterConfig] = [] + config_languages = [*languages, "base"] + config_lint_languages = [*lint_languages, "base"] - languages.append("base") - - for language in languages: - result = self.language_config.get_language_config(language) + for language in config_languages: + include_linter = language in config_lint_languages + result = self.language_config.get_language_config(language, include_linter) if result.config_data: successful_languages.append(language) linter_configs.append( @@ -212,10 +219,8 @@ def _build_pre_commit_config(self, languages: list[str]) -> BuildConfigResult: ) ) if result.linter_config.successful else None data = yaml.safe_load(result.config_data) - for config in data["repos"]: - config_data.append(config) + config_data += data["repos"] or [] - languages.remove("base") config = {"repos": config_data} version = hash_config(yaml.dump(config)) diff --git a/tests/actions/test_action.py b/tests/actions/test_action.py index d3f57cf6..9a07ac11 100644 --- a/tests/actions/test_action.py +++ b/tests/actions/test_action.py @@ -297,3 +297,64 @@ def test_that_update_secureli_handles_successful_update( update_result = action._update_secureli(always_yes=False) assert update_result.outcome == VerifyOutcome.UPDATE_SUCCEEDED + + +def test_that_prompt_get_lint_config_languages_returns_all_languages_when_always_true_option_is_true( + action: Action, + mock_echo: MagicMock, +): + mock_languages = ["RadLang", "MockLang"] + + result = action._prompt_get_lint_config_languages(mock_languages, True) + + mock_echo.confirm.assert_not_called() + assert result == mock_languages + + +def test_that_prompt_get_lint_config_languages_returns_no_languages( + action: Action, + mock_echo: MagicMock, +): + mock_languages = ["RadLang", "MockLang"] + mock_echo.confirm.return_value = False + + result = action._prompt_get_lint_config_languages(mock_languages, False) + + mock_echo.confirm.assert_called() + assert mock_echo.confirm.call_count == len(mock_languages) + assert result == [] + + +def test_that_prompt_get_lint_config_languages_returns_all_languages( + action: Action, + mock_echo: MagicMock, +): + mock_languages = ["RadLang", "MockLang"] + mock_echo.confirm.return_value = True + + result = action._prompt_get_lint_config_languages(mock_languages, False) + + mock_echo.confirm.assert_called() + assert mock_echo.confirm.call_count == len(mock_languages) + assert result == mock_languages + + +def test_that_prompt_get_lint_config_languages_returns_filtered_languages_based_on_choice( + action: Action, + mock_echo: MagicMock, +): + mock_languages = ["RadLang", "MockLang"] + + def confirm_side_effect(*args, **kwargs): + if mock_languages[0] in args[0]: + return True + else: + return False + + mock_echo.confirm.side_effect = confirm_side_effect + + result = action._prompt_get_lint_config_languages(mock_languages, False) + + mock_echo.confirm.assert_called() + assert mock_echo.confirm.call_count == len(mock_languages) + assert result == [mock_languages[0]] diff --git a/tests/application/test_main.py b/tests/application/test_main.py index 8069a092..2298a103 100644 --- a/tests/application/test_main.py +++ b/tests/application/test_main.py @@ -57,3 +57,12 @@ def test_that_app_implements_version_option( assert secureli_version() in result.stdout mock_container.init_resources.assert_not_called() mock_container.wire.assert_not_called() + + +def test_that_app_ignores_version_callback(mock_container: MagicMock): + result = CliRunner().invoke(secureli.main.app, ["scan"]) + + assert result.exit_code is 0 + assert secureli_version() not in result.stdout + mock_container.init_resources.assert_called_once() + mock_container.wire.assert_called_once() diff --git a/tests/services/test_language_config.py b/tests/services/test_language_config.py index 0538ae99..f7ac5718 100644 --- a/tests/services/test_language_config.py +++ b/tests/services/test_language_config.py @@ -5,13 +5,14 @@ from secureli.services.language_config import ( LanguageConfigService, LanguageNotSupportedError, + LoadLinterConfigsResult, ) @pytest.fixture() def mock_data_loader() -> MagicMock: mock_data_loader = MagicMock() - mock_data_loader.return_value = "a: 1" + mock_data_loader.return_value = "repos: [{ repo: 'mock-repo' }]" return mock_data_loader @@ -32,7 +33,7 @@ def test_that_language_config_service_treats_missing_templates_as_unsupported_la ): mock_data_loader.side_effect = ValueError with pytest.raises(LanguageNotSupportedError): - language_config_service.get_language_config("BadLang") + language_config_service.get_language_config("BadLang", include_linter=True) def test_that_language_config_service_treats_missing_templates_as_unsupported_language_when_checking_versions( @@ -41,43 +42,51 @@ def test_that_language_config_service_treats_missing_templates_as_unsupported_la ): mock_data_loader.side_effect = ValueError with pytest.raises(LanguageNotSupportedError): - language_config_service.get_language_config("BadLang") + language_config_service.get_language_config("BadLang", True) def test_that_version_identifiers_are_calculated_for_known_languages( language_config_service: LanguageConfigService, ): - version = language_config_service.get_language_config("Python").version + version = language_config_service.get_language_config( + "Python", include_linter=True + ).version assert version != None def test_that_language_config_service_templates_are_loaded_with_global_exclude_if_provided_multiple_patterns( language_config_service: LanguageConfigService, - mock_data_loader: MagicMock, ): - mock_data_loader.return_value = "yaml: data" language_config_service.ignored_file_patterns = [ "mock_pattern1", "mock_pattern2", ] - result = language_config_service.get_language_config("Python") + result = language_config_service.get_language_config("Python", include_linter=True) assert "exclude: ^(mock_pattern1|mock_pattern2)" in result.config_data def test_that_language_config_service_templates_are_loaded_without_exclude( language_config_service: LanguageConfigService, - mock_data_loader: MagicMock, - mock_open: MagicMock, ): - mock_data_loader.return_value = "yaml: data" language_config_service.ignored_file_patterns = [] - result = language_config_service.get_language_config("Python") + result = language_config_service.get_language_config("Python", include_linter=True) assert "exclude:" not in result.config_data +def test_that_language_config_service_templates_are_loaded_without_linter_config_if_include_linter_is_false( + language_config_service: LanguageConfigService, +): + language_config_service.ignored_file_patterns = [] + result = language_config_service.get_language_config("Python", include_linter=False) + + assert result.linter_config == LoadLinterConfigsResult( + successful=True, linter_data=[] + ) + + def test_that_language_config_service_does_nothing_when_pre_commit_settings_is_empty( language_config_service: LanguageConfigService, mock_data_loader: MagicMock, @@ -95,7 +104,7 @@ def mock_loader_side_effect(resource): mock_data_loader.side_effect = mock_loader_side_effect - result = language_config_service.get_language_config("Python") + result = language_config_service.get_language_config("Python", include_linter=True) assert "orig_arg" in result.config_data @@ -119,11 +128,59 @@ def test_that_language_config_service_language_config_does_not_get_loaded( def test_that_language_config_service_templates_are_loaded_with_global_exclude_if_provided( language_config_service: LanguageConfigService, - mock_data_loader: MagicMock, - mock_open: MagicMock, ): - mock_data_loader.return_value = "yaml: data" language_config_service.ignored_file_patterns = ["mock_pattern"] - result = language_config_service.get_language_config("Python") + result = language_config_service.get_language_config("Python", include_linter=True) assert "exclude: mock_pattern" in result.config_data + + +def test_that_calculate_combined_configuration_adds_lint_config( + language_config_service: LanguageConfigService, + mock_data_loader: MagicMock, +): + mock_scanner_config = "repos: [{ repo: 'scanner-pre-commit'}]" + mock_linter_config = "repos: [{ repo: 'linter-pre-commit'}]" + + def data_loader_side_effect(*args, **kwargs): + if "base" in args[0]: + return mock_scanner_config + elif "lint" in args[0]: + return mock_linter_config + else: + return "repos: []" + + mock_data_loader.side_effect = data_loader_side_effect + result = language_config_service._calculate_combined_configuration( + "RadLang", include_linter=True + ) + + assert result == { + "repos": [{"repo": "scanner-pre-commit"}, {"repo": "linter-pre-commit"}] + } + assert mock_data_loader.call_count == 2 + + +def test_that_calculate_combined_configuration_ignores_lint_config( + language_config_service: LanguageConfigService, + mock_data_loader: MagicMock, +): + mock_data_loader.return_value = "repos: [{ repo: 'scanner-pre-commit'}]" + result = language_config_service._calculate_combined_configuration( + "RadLang", include_linter=False + ) + + assert result == {"repos": [{"repo": "scanner-pre-commit"}]} + assert mock_data_loader.call_count == 1 + + +def test_that_calculate_combined_configuration_ignores_lint_config( + language_config_service: LanguageConfigService, + mock_data_loader: MagicMock, +): + mock_data_loader.return_value = "" + result = language_config_service._calculate_combined_configuration( + "RadLang", include_linter=False + ) + + assert result == {"repos": []} diff --git a/tests/services/test_language_support.py b/tests/services/test_language_support.py index 1e86ee82..ff2f0176 100644 --- a/tests/services/test_language_support.py +++ b/tests/services/test_language_support.py @@ -7,7 +7,7 @@ from secureli.abstractions.pre_commit import ( InstallResult, ) -from secureli.services.language_support import LanguageSupportService +from secureli.services.language_support import BuildConfigResult, LanguageSupportService from secureli.services.language_config import ( LanguageConfigService, LanguagePreCommitResult, @@ -245,7 +245,14 @@ def mock_loader_side_effect(resource): mock_data_loader.side_effect = mock_loader_side_effect - metadata = language_support_service.apply_support(["RadLang"]) + languages = ["RadLang"] + lint_languages = [*languages] + + build_config_result = language_support_service._build_pre_commit_config( + languages, lint_languages + ) + + metadata = language_support_service.apply_support(["RadLang"], build_config_result) assert metadata.security_hook_id == "baddie-finder" @@ -270,10 +277,17 @@ def test_that_language_support_throws_exception_when_language_config_file_cannot """, ) + languages = ["RadLang"] + lint_languages = [*languages] + + build_config_result = language_support_service._build_pre_commit_config( + languages, lint_languages + ) + mock_open.side_effect = IOError with raises(IOError): - language_support_service.apply_support(["RadLang"]) + language_support_service.apply_support(languages, build_config_result) def test_that_language_support_handles_invalid_language_config( @@ -292,5 +306,41 @@ def test_that_language_support_handles_invalid_language_config( ) ) - metadata = language_support_service.apply_support(["RadLang"]) + languages = ["RadLang"] + lint_languages = [*languages] + + build_config_result = language_support_service._build_pre_commit_config( + languages, lint_languages + ) + + metadata = language_support_service.apply_support(languages, build_config_result) assert metadata.security_hook_id is None + + +def test_that_language_support_handles_empty_repos_list( + language_support_service: LanguageSupportService, + mock_language_config_service: MagicMock, + mock_data_loader: MagicMock, +): + mock_language_config_service.get_language_config.return_value = LanguagePreCommitResult( + language="Python", + version="abc123", + linter_config=LoadLinterConfigsResult( + successful=True, + linter_data=[{"key": {"example"}}], + ), + config_data=""" + repos: + """, + ) + + mock_data_loader.return_value = "" + + languages = ["RadLang"] + lint_languages = [*languages] + + build_config_result = language_support_service._build_pre_commit_config( + languages, lint_languages + ) + + assert build_config_result.config_data["repos"] == []