From 35b18e95b00d7006683d72a41f229028319309ae Mon Sep 17 00:00:00 2001 From: Kevin Orlando Date: Thu, 14 Dec 2023 22:23:54 -0500 Subject: [PATCH 1/4] fix config file generation issue, update config templates --- .../files/configs/javascript.config.yaml | 12 +++++----- .../files/configs/typescript.config.yaml | 10 ++++---- secureli/services/language_support.py | 24 +++++++++++++------ 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/secureli/resources/files/configs/javascript.config.yaml b/secureli/resources/files/configs/javascript.config.yaml index 7d61b675..4d4cf999 100644 --- a/secureli/resources/files/configs/javascript.config.yaml +++ b/secureli/resources/files/configs/javascript.config.yaml @@ -1,6 +1,6 @@ -eslintrc: - extends: ["google"] - env: - es6: true - plugins: - - prettier +filename: ".eslintrc.yaml" +data: + extends: ["google"] + env: + es6: true + plugins: ["prettier"] diff --git a/secureli/resources/files/configs/typescript.config.yaml b/secureli/resources/files/configs/typescript.config.yaml index 4afe7c51..e711754d 100644 --- a/secureli/resources/files/configs/typescript.config.yaml +++ b/secureli/resources/files/configs/typescript.config.yaml @@ -1,5 +1,5 @@ -eslintrc: - extends: ['google', 'plugin:@typescript-eslint/recommended', 'prettier'] - parser: '@typescript-eslint/parser' - plugins: - - '@typescript-eslint' +filename: ".eslintrc.yaml" +data: + extends: ["google", "plugin:@typescript-eslint/recommended", "prettier"] + parser: "@typescript-eslint/parser" + plugins: ["@typescript-eslint"] diff --git a/secureli/services/language_support.py b/secureli/services/language_support.py index 18fb3ed8..08890229 100644 --- a/secureli/services/language_support.py +++ b/secureli/services/language_support.py @@ -252,15 +252,25 @@ def _write_pre_commit_configs( # parse though each config for the given language. for config in language_linter_configs.linter_data: try: - config_name = list(config.keys())[0] # generate relative file name and path. - config_file_name = f"{slugify(language_linter_configs.language)}.{config_name}.yaml" - path_to_config_file = ( - SecureliConfig.FOLDER_PATH / ".secureli/{config_file_name}" + config_file_name = ( + f"{slugify(language_linter_configs.language)}.config.yaml" ) - with open(path_to_config_file, "w") as f: - f.write(yaml.dump(config[config_name])) - num_configs_success += 1 + + absolute_secureli_path = ( + f'{Path(f"{__file__}").parent.resolve()}'.rsplit("/", 1)[0] + ) + + path_to_config_file = Path( + f"{absolute_secureli_path}/resources/files/configs/{config_file_name}" + ) + + if Path.exists(path_to_config_file): + with open( + Path(SecureliConfig.FOLDER_PATH / config["filename"]), "w" + ) as f: + f.write(yaml.dump(config["data"])) + num_configs_success += 1 except Exception as e: num_configs_non_success += 1 non_success_messages.append(f"Unable to install config: {e}") From 9d1522ebaabb4fc9e28037810dc093d5df276b17 Mon Sep 17 00:00:00 2001 From: Kevin Orlando Date: Thu, 21 Dec 2023 10:38:07 -0500 Subject: [PATCH 2/4] update lint config writer logic; add unit tests --- secureli/container.py | 1 + secureli/services/language_config.py | 1 - secureli/services/language_support.py | 69 +++++++++------------ tests/services/test_language_support.py | 82 +++++++++++++++++++++++-- 4 files changed, 107 insertions(+), 46 deletions(-) diff --git a/secureli/container.py b/secureli/container.py index f856b626..285717c8 100644 --- a/secureli/container.py +++ b/secureli/container.py @@ -108,6 +108,7 @@ class Container(containers.DeclarativeContainer): git_ignore=git_ignore_service, language_config=language_config_service, data_loader=read_resource, + echo=echo, ) """Analyzes a given repo to try to identify the most common language""" diff --git a/secureli/services/language_config.py b/secureli/services/language_config.py index 34c9c011..0c4b701f 100644 --- a/secureli/services/language_config.py +++ b/secureli/services/language_config.py @@ -7,7 +7,6 @@ from secureli.resources.slugify import slugify from secureli.utilities.hash import hash_config from secureli.utilities.patterns import combine_patterns -import secureli.repositories.secureli_config as SecureliConfig class LanguageNotSupportedError(Exception): diff --git a/secureli/services/language_support.py b/secureli/services/language_support.py index 08890229..c573ac66 100644 --- a/secureli/services/language_support.py +++ b/secureli/services/language_support.py @@ -3,6 +3,7 @@ import pydantic import yaml +from secureli.abstractions.echo import EchoAbstraction import secureli.repositories.secureli_config as SecureliConfig from secureli.abstractions.pre_commit import PreCommitAbstraction @@ -68,9 +69,18 @@ class UnexpectedReposResult(pydantic.BaseModel): unexpected_repos: Optional[list[str]] = [] +class LinterConfigData(pydantic.BaseModel): + """ + Represents the structure of a linter config file + """ + + filename: str + data: Any + + class LinterConfig(pydantic.BaseModel): language: str - linter_data: list[Any] + linter_data: list[LinterConfigData] class BuildConfigResult(pydantic.BaseModel): @@ -95,11 +105,13 @@ def __init__( language_config: LanguageConfigService, git_ignore: GitIgnoreService, data_loader: Callable[[str], str], + echo: EchoAbstraction, ): self.git_ignore = git_ignore self.pre_commit_hook = pre_commit_hook self.language_config = language_config self.data_loader = data_loader + self.echo = echo def apply_support( self, languages: list[str], language_config_result: BuildConfigResult @@ -232,51 +244,28 @@ def _build_pre_commit_config( linter_configs=linter_configs, ) - @staticmethod def _write_pre_commit_configs( + self, all_linter_configs: list[LinterConfig], - ) -> LanguageLinterWriteResult: + ) -> None: """ Install any config files for given language to support any pre-commit commands. i.e. Javascript ESLint requires a .eslintrc file to sufficiently use plugins and allow for further customization for repo's flavor of Javascript - :return: LanguageLinterWriteResult + :param all_linter_configs: the applicable linter configs to create config files for in the repo """ - num_configs_success = 0 - num_configs_non_success = 0 - non_success_messages = list[str]() - - # parse through languages for their linter config if any. - for language_linter_configs in all_linter_configs: - # parse though each config for the given language. - for config in language_linter_configs.linter_data: - try: - # generate relative file name and path. - config_file_name = ( - f"{slugify(language_linter_configs.language)}.config.yaml" - ) - - absolute_secureli_path = ( - f'{Path(f"{__file__}").parent.resolve()}'.rsplit("/", 1)[0] - ) - - path_to_config_file = Path( - f"{absolute_secureli_path}/resources/files/configs/{config_file_name}" - ) + linter_config_data = [ + (linter_data, config.language) + for config in all_linter_configs + for linter_data in config.linter_data + ] - if Path.exists(path_to_config_file): - with open( - Path(SecureliConfig.FOLDER_PATH / config["filename"]), "w" - ) as f: - f.write(yaml.dump(config["data"])) - num_configs_success += 1 - except Exception as e: - num_configs_non_success += 1 - non_success_messages.append(f"Unable to install config: {e}") - - return LanguageLinterWriteResult( - num_successful=num_configs_success, - num_non_success=num_configs_non_success, - non_success_messages=non_success_messages, - ) + for config, language in linter_config_data: + try: + with open(Path(SecureliConfig.FOLDER_PATH / config.filename), "w") as f: + f.write(yaml.dump(config.data)) + except: + self.echo.warning( + f"Failed to write {config.filename} config file for {language}" + ) diff --git a/tests/services/test_language_support.py b/tests/services/test_language_support.py index ff2f0176..bcf7cd21 100644 --- a/tests/services/test_language_support.py +++ b/tests/services/test_language_support.py @@ -7,7 +7,11 @@ from secureli.abstractions.pre_commit import ( InstallResult, ) -from secureli.services.language_support import BuildConfigResult, LanguageSupportService +from secureli.services.language_support import ( + LanguageSupportService, + LinterConfig, + LinterConfigData, +) from secureli.services.language_config import ( LanguageConfigService, LanguagePreCommitResult, @@ -32,6 +36,8 @@ def mock_open_config(mocker: MockerFixture): """ ) mocker.patch("builtins.open", mock_open) + mock_open.return_value.write = MagicMock() + return mock_open @pytest.fixture() @@ -77,6 +83,12 @@ def mock_git_ignore() -> MagicMock: return mock_git_ignore +@pytest.fixture() +def mock_echo() -> MagicMock: + mock_echo = MagicMock() + return mock_echo + + @pytest.fixture() def mock_language_config_service() -> LanguageConfigService: mock_language_config_service = MagicMock() @@ -90,12 +102,14 @@ def language_support_service( mock_git_ignore: MagicMock, mock_language_config_service: MagicMock, mock_data_loader: MagicMock, + mock_echo: MagicMock, ) -> LanguageSupportService: return LanguageSupportService( pre_commit_hook=mock_pre_commit_hook, git_ignore=mock_git_ignore, language_config=mock_language_config_service, data_loader=mock_data_loader, + echo=mock_echo, ) @@ -233,7 +247,7 @@ def mock_loader_side_effect(resource): version="abc123", linter_config=LoadLinterConfigsResult( successful=True, - linter_data=[{"key": {"example"}}], + linter_data=[{"filename": "test.txt", "data": {}}], ), config_data=""" repos: @@ -267,7 +281,7 @@ def test_that_language_support_throws_exception_when_language_config_file_cannot version="abc123", linter_config=LoadLinterConfigsResult( successful=True, - linter_data=[{"key": {"example"}}], + linter_data=[{"filename": "test.txt", "data": {}}], ), config_data=""" repos: @@ -300,7 +314,7 @@ def test_that_language_support_handles_invalid_language_config( version="abc123", linter_config=LoadLinterConfigsResult( successful=True, - linter_data=[{"key": {"example"}}], + linter_data=[{"filename": "test.txt", "data": {}}], ), config_data="", ) @@ -327,7 +341,7 @@ def test_that_language_support_handles_empty_repos_list( version="abc123", linter_config=LoadLinterConfigsResult( successful=True, - linter_data=[{"key": {"example"}}], + linter_data=[{"filename": "test.txt", "data": {}}], ), config_data=""" repos: @@ -344,3 +358,61 @@ def test_that_language_support_handles_empty_repos_list( ) assert build_config_result.config_data["repos"] == [] + + +def test_write_pre_commit_configs_writes_successfully( + language_support_service: LanguageSupportService, + mock_open: MagicMock, + mock_echo: MagicMock, +): + configs = [ + LinterConfig( + language="RadLag", + linter_data=[LinterConfigData(filename="rad-lint.yml", data={})], + ), + LinterConfig( + language="CoolLang", + linter_data=[LinterConfigData(filename="cool-lint.yml", data={})], + ), + ] + language_support_service._write_pre_commit_configs(configs) + + assert mock_open.call_count == len(configs) + assert mock_open.return_value.write.call_count == len(configs) + mock_echo.warning.assert_not_called() + + +def test_write_pre_commit_configs_ignores_empty_linter_arr( + language_support_service: LanguageSupportService, + mock_open: MagicMock, + mock_echo: MagicMock, +): + language_support_service._write_pre_commit_configs([]) + + mock_open.assert_not_called() + mock_open.return_value.write.assert_not_called() + mock_echo.warning.assert_not_called() + + +def test_write_pre_commit_configs_ignores_exceptions( + language_support_service: LanguageSupportService, + mock_open: MagicMock, + mock_echo: MagicMock, +): + mock_open.side_effect = Exception("error") + mock_language = "CoolLang" + mock_filename = "cool-lint-config.yml" + language_support_service._write_pre_commit_configs( + [ + LinterConfig( + language=mock_language, + linter_data=[LinterConfigData(filename=mock_filename, data={})], + ), + ] + ) + + mock_open.assert_called_once() + mock_open.return_value.write.assert_not_called() + mock_echo.warning.assert_called_once_with( + f"Failed to write {mock_filename} config file for {mock_language}" + ) From 155fb826703037c9bd85a58f128eea009c4f302a Mon Sep 17 00:00:00 2001 From: Kevin Orlando Date: Thu, 21 Dec 2023 14:28:19 -0500 Subject: [PATCH 3/4] change test name --- tests/services/test_language_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/services/test_language_support.py b/tests/services/test_language_support.py index bcf7cd21..d7532432 100644 --- a/tests/services/test_language_support.py +++ b/tests/services/test_language_support.py @@ -394,7 +394,7 @@ def test_write_pre_commit_configs_ignores_empty_linter_arr( mock_echo.warning.assert_not_called() -def test_write_pre_commit_configs_ignores_exceptions( +def test_write_pre_commit_configs_handle_exceptions( language_support_service: LanguageSupportService, mock_open: MagicMock, mock_echo: MagicMock, From 13502e6a929a1d4f34dba8fa3a6ac9d94ebf51b4 Mon Sep 17 00:00:00 2001 From: Kevin Orlando Date: Thu, 21 Dec 2023 21:51:06 -0500 Subject: [PATCH 4/4] change key name to settings --- .../resources/files/configs/javascript.config.yaml | 2 +- .../resources/files/configs/typescript.config.yaml | 2 +- secureli/services/language_support.py | 4 ++-- tests/services/test_language_support.py | 14 +++++++------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/secureli/resources/files/configs/javascript.config.yaml b/secureli/resources/files/configs/javascript.config.yaml index 4d4cf999..7660772b 100644 --- a/secureli/resources/files/configs/javascript.config.yaml +++ b/secureli/resources/files/configs/javascript.config.yaml @@ -1,5 +1,5 @@ filename: ".eslintrc.yaml" -data: +settings: extends: ["google"] env: es6: true diff --git a/secureli/resources/files/configs/typescript.config.yaml b/secureli/resources/files/configs/typescript.config.yaml index e711754d..85978b15 100644 --- a/secureli/resources/files/configs/typescript.config.yaml +++ b/secureli/resources/files/configs/typescript.config.yaml @@ -1,5 +1,5 @@ filename: ".eslintrc.yaml" -data: +settings: extends: ["google", "plugin:@typescript-eslint/recommended", "prettier"] parser: "@typescript-eslint/parser" plugins: ["@typescript-eslint"] diff --git a/secureli/services/language_support.py b/secureli/services/language_support.py index c573ac66..b1f0567e 100644 --- a/secureli/services/language_support.py +++ b/secureli/services/language_support.py @@ -75,7 +75,7 @@ class LinterConfigData(pydantic.BaseModel): """ filename: str - data: Any + settings: Any class LinterConfig(pydantic.BaseModel): @@ -264,7 +264,7 @@ def _write_pre_commit_configs( for config, language in linter_config_data: try: with open(Path(SecureliConfig.FOLDER_PATH / config.filename), "w") as f: - f.write(yaml.dump(config.data)) + f.write(yaml.dump(config.settings)) except: self.echo.warning( f"Failed to write {config.filename} config file for {language}" diff --git a/tests/services/test_language_support.py b/tests/services/test_language_support.py index d7532432..1d6a555d 100644 --- a/tests/services/test_language_support.py +++ b/tests/services/test_language_support.py @@ -247,7 +247,7 @@ def mock_loader_side_effect(resource): version="abc123", linter_config=LoadLinterConfigsResult( successful=True, - linter_data=[{"filename": "test.txt", "data": {}}], + linter_data=[{"filename": "test.txt", "settings": {}}], ), config_data=""" repos: @@ -281,7 +281,7 @@ def test_that_language_support_throws_exception_when_language_config_file_cannot version="abc123", linter_config=LoadLinterConfigsResult( successful=True, - linter_data=[{"filename": "test.txt", "data": {}}], + linter_data=[{"filename": "test.txt", "settings": {}}], ), config_data=""" repos: @@ -314,7 +314,7 @@ def test_that_language_support_handles_invalid_language_config( version="abc123", linter_config=LoadLinterConfigsResult( successful=True, - linter_data=[{"filename": "test.txt", "data": {}}], + linter_data=[{"filename": "test.txt", "settings": {}}], ), config_data="", ) @@ -341,7 +341,7 @@ def test_that_language_support_handles_empty_repos_list( version="abc123", linter_config=LoadLinterConfigsResult( successful=True, - linter_data=[{"filename": "test.txt", "data": {}}], + linter_data=[{"filename": "test.txt", "settings": {}}], ), config_data=""" repos: @@ -368,11 +368,11 @@ def test_write_pre_commit_configs_writes_successfully( configs = [ LinterConfig( language="RadLag", - linter_data=[LinterConfigData(filename="rad-lint.yml", data={})], + linter_data=[LinterConfigData(filename="rad-lint.yml", settings={})], ), LinterConfig( language="CoolLang", - linter_data=[LinterConfigData(filename="cool-lint.yml", data={})], + linter_data=[LinterConfigData(filename="cool-lint.yml", settings={})], ), ] language_support_service._write_pre_commit_configs(configs) @@ -406,7 +406,7 @@ def test_write_pre_commit_configs_handle_exceptions( [ LinterConfig( language=mock_language, - linter_data=[LinterConfigData(filename=mock_filename, data={})], + linter_data=[LinterConfigData(filename=mock_filename, settings={})], ), ] )