From d6cdb4761789cf3809fd42204e53bff2b4ab0582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D1=8C=D1=8F=20=D0=9B=D0=B5=D0=B1=D0=B5=D0=B4?= =?UTF-8?q?=D0=B5=D0=B2?= Date: Mon, 29 Jul 2019 19:33:12 +0300 Subject: [PATCH] add skip dotfiles and dirs behavior --- README.md | 1 + rozental_as_a_service/args_utils.py | 3 +++ rozental_as_a_service/common_types.py | 1 + rozental_as_a_service/config_utils.py | 2 ++ rozental_as_a_service/rozental.py | 10 +++++++++- tests/test_integrational.py | 2 +- 6 files changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 472d0bc..1fac834 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ - `--db_path` – путь до sqlite-базы данных с кэшем для Розенталя. По-умолчанию создаётся `.rozental.sqlite` в директории для проверки. - `--exclude` – список каталогов, в которых не нужно проверять файлы. Например, `tests/,cache/,lib/,dist/`. - `--exit_zero` – в любом случае завершать процесс без ошибки. Пригодится, если вы не хотите ломать билд при наличии опечаток (полезно при внедрении). +- `--process_dots` – проверять файлы и директории, название которых начинается с точки. По-умолчанию они пропускаются. - `--processes` – количество процессов, которые будут использоваться для извлечения строк. По-умолчанию используется доступное количество процессоров. - `-v` – более многословный режим. diff --git a/rozental_as_a_service/args_utils.py b/rozental_as_a_service/args_utils.py index 7a3a97e..659e7ac 100644 --- a/rozental_as_a_service/args_utils.py +++ b/rozental_as_a_service/args_utils.py @@ -17,6 +17,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument('--exclude', default='') parser.add_argument('--db_path') parser.add_argument('--exit_zero', action='store_true') + parser.add_argument('--process_dots', action='store_true', help='process dot-files and dot-directories') parser.add_argument('--processes', type=int, default=None) parser.add_argument('-v', action='count', default=0) @@ -43,6 +44,7 @@ def prepare_arguments(argparse_args: argparse.Namespace) -> RozentalOptions: exclude = argparse_args.exclude.split(',') if argparse_args.exclude else config.get('exclude', []) exit_zero = argparse_args.exit_zero or config.get('exit_zero') or False verbosity = argparse_args.v or config.get('v') or 0 + process_dots = argparse_args.process_dots or config.get('process_dots') or False return { 'path': argparse_args.path, @@ -50,6 +52,7 @@ def prepare_arguments(argparse_args: argparse.Namespace) -> RozentalOptions: 'exclude': exclude, 'db_path': db_path, 'exit_zero': exit_zero, + 'process_dots': process_dots, 'processes_amount': processes_amount, 'verbosity': verbosity, } diff --git a/rozental_as_a_service/common_types.py b/rozental_as_a_service/common_types.py index 89b7543..76ecfbb 100644 --- a/rozental_as_a_service/common_types.py +++ b/rozental_as_a_service/common_types.py @@ -19,5 +19,6 @@ class RozentalOptions(TypedDict): exclude: List[str] db_path: str exit_zero: bool + process_dots: bool processes_amount: int verbosity: int diff --git a/rozental_as_a_service/config_utils.py b/rozental_as_a_service/config_utils.py index 5f07225..da079bb 100644 --- a/rozental_as_a_service/config_utils.py +++ b/rozental_as_a_service/config_utils.py @@ -16,6 +16,8 @@ def get_params_from_config(config_path: str) -> Mapping[str, Any]: params['exclude'] = params['exclude'].split(',') # type: ignore if 'exit_zero' in params: params['exit_zero'] = params['exit_zero'] == 'True' # type: ignore + if 'process_dots' in params: + params['process_dots'] = params['process_dots'] == 'True' # type: ignore if 'verbosity' in params: params['verbosity'] = int(params['verbosity']) # type: ignore return params diff --git a/rozental_as_a_service/rozental.py b/rozental_as_a_service/rozental.py index 764a4cb..8076740 100644 --- a/rozental_as_a_service/rozental.py +++ b/rozental_as_a_service/rozental.py @@ -28,7 +28,12 @@ logging.getLogger('urllib3').setLevel(logging.INFO) -def extract_all_constants_from_path(path: str, exclude: List[str], processes_amount: int) -> List[str]: +def extract_all_constants_from_path( + path: str, + exclude: List[str], + process_dots: bool, + processes_amount: int, +) -> List[str]: extractors = [ (extract_from_python_src, ['py', 'pyi']), (extract_from_markdown, ['md']), @@ -46,6 +51,8 @@ def extract_all_constants_from_path(path: str, exclude: List[str], processes_amo for extension, extension_extractors in extension_to_extractor_mapping.items(): all_files = get_all_filepathes_recursively(path, exclude, extension) + if not process_dots: + all_files = [f for f in all_files if '/.' not in f] if not all_files: continue chunk_size = math.ceil(len(all_files) / processes_amount) @@ -125,6 +132,7 @@ def main() -> None: unique_words = extract_all_constants_from_path( arguments['path'], arguments['exclude'], + arguments['process_dots'], arguments['processes_amount'], ) typos_info = fetch_typos_info(unique_words, arguments['vocabulary_path'], arguments['db_path']) diff --git a/tests/test_integrational.py b/tests/test_integrational.py index fa303a9..30220b1 100644 --- a/tests/test_integrational.py +++ b/tests/test_integrational.py @@ -2,7 +2,7 @@ def test_finds_correct_py_files_typos(): - unique_words = extract_all_constants_from_path('tests/test_files/', [], processes_amount=2) + unique_words = extract_all_constants_from_path('tests/test_files/', [], process_dots=False, processes_amount=2) typos_info = fetch_typos_info(unique_words, None, None) expected_typos = ['бджета', 'ркеламную', 'содание'] assert sorted(t['original'] for t in typos_info) == expected_typos