From dd553dcf8d6e71e33a17e866dff6156dbd9e204d Mon Sep 17 00:00:00 2001 From: Abhishek Mohan Date: Tue, 1 Aug 2023 16:45:10 -0400 Subject: [PATCH] Flag invalid select/exclude using custom mode (#428) ## Description Throws an error if select/exclude filters are not valid. Validation: Must start with "path:", "tag:", or "config.materialized", "config.schema", or "config.tags" ## Checklist - [ ] I have made corresponding changes to the documentation (if required) - [ ] I have added tests that prove my fix is effective or that my feature works --- cosmos/dbt/selector.py | 13 +++++++++++++ tests/dbt/test_selector.py | 10 +++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index ec1cd6ae5..2a219d59d 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -4,6 +4,8 @@ from typing import TYPE_CHECKING +from cosmos.exceptions import CosmosValueError + if TYPE_CHECKING: from cosmos.dbt.graph import DbtNode @@ -132,6 +134,17 @@ def select_nodes( if not select and not exclude: return nodes + # validates select and exclude filters + filters = [["select", select], ["exclude", exclude]] + for filter_type, filter in filters: + for filter_parameter in filter: + if filter_parameter.startswith(PATH_SELECTOR) or filter_parameter.startswith(TAG_SELECTOR): + continue + elif any([filter_parameter.startswith(CONFIG_SELECTOR + config + ":") for config in SUPPORTED_CONFIG]): + continue + else: + raise CosmosValueError(f"Invalid {filter_type} filter: {filter_parameter}") + subset_ids: set[str] = set() for statement in select: diff --git a/tests/dbt/test_selector.py b/tests/dbt/test_selector.py index 6baaf5c89..57616628c 100644 --- a/tests/dbt/test_selector.py +++ b/tests/dbt/test_selector.py @@ -1,8 +1,11 @@ from pathlib import Path +import pytest + from cosmos.constants import DbtResourceType from cosmos.dbt.graph import DbtNode from cosmos.dbt.selector import select_nodes +from cosmos.exceptions import CosmosValueError SAMPLE_PROJ_PATH = Path("/home/user/path/dbt-proj/") @@ -82,9 +85,10 @@ def test_select_nodes_by_exclude_tag(): assert selected == expected -def test_select_nodes_by_exclude_unsupported_selector(caplog): - select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, exclude=["unsupported:filter"]) - assert "Unsupported select statement: unsupported:filter" in caplog.messages +def test_select_nodes_by_exclude_unsupported_selector(): + with pytest.raises(CosmosValueError) as err_info: + assert select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, exclude=["unsupported:filter"]) + assert err_info.value.args[0] == "Invalid exclude filter: unsupported:filter" def test_select_nodes_by_select_union_exclude_tags():