Skip to content

Commit

Permalink
Support building DbtDag without setting paths in ProjectConfig (#…
Browse files Browse the repository at this point in the history
…1307)

Previously, users were unable to run a DbtDag or DbtTask group where the
following was used:
- `RenderMode.DBT_LS`
- `ExecutionConfig(dbt_project_path)`
- `RenderConfig(dbt_project_path)`

This scenario can be helpful when using ExecutionMode.KUBERNETES or
other similar ones and was found out during:
#1297
  • Loading branch information
tatiana authored Nov 6, 2024
1 parent c9ea2a2 commit 739db95
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 3 deletions.
7 changes: 7 additions & 0 deletions cosmos/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,13 @@ def __post_init__(self, dbt_project_path: str | Path | None) -> None:
# allows us to initiate this attribute from Path objects and str
self.dbt_ls_path = Path(self.dbt_ls_path) if self.dbt_ls_path else None

@property
def project_name(self) -> str:
if self.project_path:
return Path(self.project_path).stem
else:
return ""

def validate_dbt_command(self, fallback_cmd: str | Path = "") -> None:
"""
When using LoadMode.DBT_LS, the dbt executable path is necessary for rendering.
Expand Down
6 changes: 3 additions & 3 deletions cosmos/converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def validate_initial_user_config(
)


def validate_adapted_user_config(
def validate_changed_config_paths(
execution_config: ExecutionConfig | None, project_config: ProjectConfig, render_config: RenderConfig | None
):
"""
Expand Down Expand Up @@ -226,7 +226,7 @@ def __init__(
render_config.project_path = project_config.dbt_project_path
execution_config.project_path = project_config.dbt_project_path

validate_adapted_user_config(execution_config, project_config, render_config)
validate_changed_config_paths(execution_config, project_config, render_config)

env_vars = copy.deepcopy(project_config.env_vars or operator_args.get("env"))
dbt_vars = copy.deepcopy(project_config.dbt_vars or operator_args.get("vars"))
Expand Down Expand Up @@ -298,7 +298,7 @@ def __init__(
execution_mode=execution_config.execution_mode,
task_args=task_args,
test_indirect_selection=execution_config.test_indirect_selection,
dbt_project_name=project_config.project_name,
dbt_project_name=render_config.project_name,
on_warning_callback=on_warning_callback,
render_config=render_config,
)
Expand Down
35 changes: 35 additions & 0 deletions tests/test_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,41 @@ def test_converter_multiple_calls_same_operator_args(
assert operator_args == original_operator_args


@patch("cosmos.config.ProjectConfig.validate_project")
@patch("cosmos.converter.build_airflow_graph")
@patch("cosmos.converter.DbtGraph.load")
def test_validate_converter_fetches_project_name_from_render_config(
mock_dbt_graph_load, mock_build_airflow_graph, mock_validate_project
):
"""
Allow DbtToAirflowConverter to work when using:
- RenderMode.DBT_LS
- ExecutionConfig(dbt_project_path)
- RenderConfig(dbt_project_path)
In other words, when ProjectConfig does not contain name.
This scenario can be useful when using ExecutionMode.KUBERNETES or other similar ones and was found out during:
https://github.com/astronomer/astronomer-cosmos/pull/1297
"""
execution_config = ExecutionConfig(dbt_project_path="/data/project1")
project_config = ProjectConfig()
profile_config = MagicMock()
render_config = RenderConfig(dbt_project_path="/home/usr/airflow/project1")

with DAG("test-id", start_date=datetime(2022, 1, 1)) as dag:
DbtToAirflowConverter(
dag=dag,
nodes=nodes,
project_config=project_config,
profile_config=profile_config,
execution_config=execution_config,
render_config=render_config,
)

mock_build_airflow_graph.assert_called_once()
assert mock_build_airflow_graph.call_args.kwargs["dbt_project_name"] == "project1"


@pytest.mark.parametrize("invocation_mode", [None, InvocationMode.SUBPROCESS, InvocationMode.DBT_RUNNER])
@patch("cosmos.config.ProjectConfig.validate_project")
@patch("cosmos.converter.validate_initial_user_config")
Expand Down

0 comments on commit 739db95

Please sign in to comment.