Skip to content

Commit

Permalink
Flag invalid select/exclude using custom mode (#428)
Browse files Browse the repository at this point in the history
## Description

Throws an error if select/exclude filters are not valid.

Validation:
Must start with "path:", "tag:", or "config.materialized",
"config.schema", or "config.tags"

## Checklist

- [ ] I have made corresponding changes to the documentation (if
required)
- [ ] I have added tests that prove my fix is effective or that my
feature works
  • Loading branch information
Abhishek Mohan authored Aug 1, 2023
1 parent a9e3659 commit dd553dc
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 3 deletions.
13 changes: 13 additions & 0 deletions cosmos/dbt/selector.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

from typing import TYPE_CHECKING

from cosmos.exceptions import CosmosValueError

if TYPE_CHECKING:
from cosmos.dbt.graph import DbtNode

Expand Down Expand Up @@ -132,6 +134,17 @@ def select_nodes(
if not select and not exclude:
return nodes

# validates select and exclude filters
filters = [["select", select], ["exclude", exclude]]
for filter_type, filter in filters:
for filter_parameter in filter:
if filter_parameter.startswith(PATH_SELECTOR) or filter_parameter.startswith(TAG_SELECTOR):
continue
elif any([filter_parameter.startswith(CONFIG_SELECTOR + config + ":") for config in SUPPORTED_CONFIG]):
continue
else:
raise CosmosValueError(f"Invalid {filter_type} filter: {filter_parameter}")

subset_ids: set[str] = set()

for statement in select:
Expand Down
10 changes: 7 additions & 3 deletions tests/dbt/test_selector.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
from pathlib import Path

import pytest

from cosmos.constants import DbtResourceType
from cosmos.dbt.graph import DbtNode
from cosmos.dbt.selector import select_nodes
from cosmos.exceptions import CosmosValueError

SAMPLE_PROJ_PATH = Path("/home/user/path/dbt-proj/")

Expand Down Expand Up @@ -82,9 +85,10 @@ def test_select_nodes_by_exclude_tag():
assert selected == expected


def test_select_nodes_by_exclude_unsupported_selector(caplog):
select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, exclude=["unsupported:filter"])
assert "Unsupported select statement: unsupported:filter" in caplog.messages
def test_select_nodes_by_exclude_unsupported_selector():
with pytest.raises(CosmosValueError) as err_info:
assert select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, exclude=["unsupported:filter"])
assert err_info.value.args[0] == "Invalid exclude filter: unsupported:filter"


def test_select_nodes_by_select_union_exclude_tags():
Expand Down

0 comments on commit dd553dc

Please sign in to comment.