diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/cli/app.py b/python_modules/libraries/dagster-dbt/dagster_dbt/cli/app.py index 0f8a349418e5a..5546e01abb83c 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/cli/app.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/cli/app.py @@ -73,6 +73,7 @@ def copy_scaffold( project_name: str, dagster_project_dir: Path, dbt_project_dir: Path, + use_dbt_project_package_data_dir: bool, ) -> None: shutil.copytree(src=STARTER_PROJECT_PATH, dst=dagster_project_dir) dagster_project_dir.joinpath("__init__.py").unlink() @@ -89,6 +90,9 @@ def copy_scaffold( for target in profile["outputs"].values() ] + if use_dbt_project_package_data_dir: + dbt_project_dir = dagster_project_dir.joinpath("dbt-project") + dbt_project_dir_relative_path = Path( os.path.relpath( dbt_project_dir, @@ -118,6 +122,7 @@ def copy_scaffold( dbt_assets_name=f"{dbt_project_name}_dbt_assets", dbt_adapter_packages=dbt_adapter_packages, project_name=project_name, + use_dbt_project_package_data_dir=use_dbt_project_package_data_dir, ).dump(destination_path) path.unlink() @@ -154,6 +159,15 @@ def project_scaffold_command( resolve_path=True, ), ] = Path.cwd(), + use_dbt_project_package_data_dir: Annotated[ + bool, + typer.Option( + default=..., + help="Controls whether the dbt project package data directory is used.", + is_flag=True, + hidden=True, + ), + ] = False, ) -> None: """This command will initialize a new Dagster project and create directories and files that load assets from an existing dbt project. @@ -173,6 +187,7 @@ def project_scaffold_command( project_name=project_name, dagster_project_dir=dagster_project_dir, dbt_project_dir=dbt_project_dir, + use_dbt_project_package_data_dir=use_dbt_project_package_data_dir, ) console.print( diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/include/scaffold/definitions.py.jinja b/python_modules/libraries/dagster-dbt/dagster_dbt/include/scaffold/definitions.py.jinja index a02bba7e1fdeb..beffbf4a8105d 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/include/scaffold/definitions.py.jinja +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/include/scaffold/definitions.py.jinja @@ -4,6 +4,10 @@ from pathlib import Path from dagster import Definitions, OpExecutionContext from dagster_dbt import DbtCliResource, build_schedule_from_dbt_selection, dbt_assets +{% if use_dbt_project_package_data_dir -%} +# We expect the dbt project to be installed as package data. +# For details, see https://docs.python.org/3/distutils/setupscript.html#installing-package-data. +{%- endif %} dbt_project_dir = Path(__file__).parent.joinpath({{ dbt_project_dir_relative_path_parts | join(', ')}}) dbt = DbtCliResource(project_dir=os.fspath(dbt_project_dir)) diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/include/setup.py.jinja b/python_modules/libraries/dagster-dbt/dagster_dbt/include/setup.py.jinja index 53064fe2574a0..4956224aeda95 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/include/setup.py.jinja +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/include/setup.py.jinja @@ -4,6 +4,13 @@ setup( name="{{ project_name }}", version="0.0.1", packages=find_packages(), + {% if use_dbt_project_package_data_dir -%} + package_data={ + "{{ project_name }}": [ + "dbt-project/**/*", + ], + }, + {%- endif %} install_requires=[ "dagster", "dagster-cloud", diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/cli/test_app.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/cli/test_app.py index f0b3185c69dcc..217cac178161e 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/cli/test_app.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/cli/test_app.py @@ -33,12 +33,16 @@ def dbt_project_dir_fixture(tmp_path: Path) -> Path: return dbt_project_dir +@pytest.mark.parametrize("use_dbt_project_package_data_dir", [True, False]) def test_project_scaffold_command_with_precompiled_manifest( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, dbt_project_dir: Path + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + dbt_project_dir: Path, + use_dbt_project_package_data_dir: bool, ) -> None: monkeypatch.chdir(tmp_path) - project_name = "test_dagster_scaffold" + project_name = f"test_dagster_scaffold_{use_dbt_project_package_data_dir}" dagster_project_dir = tmp_path.joinpath(project_name) result = runner.invoke( @@ -50,6 +54,7 @@ def test_project_scaffold_command_with_precompiled_manifest( project_name, "--dbt-project-dir", os.fspath(dbt_project_dir), + *(["--use-dbt-project-package-data-dir"] if use_dbt_project_package_data_dir else []), ], ) @@ -61,6 +66,13 @@ def test_project_scaffold_command_with_precompiled_manifest( assert not any(path.suffix == ".jinja" for path in dagster_project_dir.glob("**/*")) assert "dbt-duckdb" in dagster_project_dir.joinpath("setup.py").read_text() + if use_dbt_project_package_data_dir: + dbt_project_dir = dagster_project_dir.joinpath("dbt-project") + shutil.copytree( + src=test_dagster_metadata_dbt_project_path, + dst=dbt_project_dir, + ) + subprocess.run(["dbt", "compile"], cwd=dbt_project_dir, check=True) assert dbt_project_dir.joinpath("target", "manifest.json").exists() @@ -82,12 +94,16 @@ def test_project_scaffold_command_with_precompiled_manifest( assert materialize_dbt_models_schedule.cron_schedule == "0 0 * * *" +@pytest.mark.parametrize("use_dbt_project_package_data_dir", [True, False]) def test_project_scaffold_command_with_runtime_manifest( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, dbt_project_dir: Path + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + dbt_project_dir: Path, + use_dbt_project_package_data_dir: bool, ) -> None: monkeypatch.chdir(tmp_path) - project_name = "test_dagster_scaffold_runtime_manifest" + project_name = f"test_dagster_scaffold_runtime_manifest_{use_dbt_project_package_data_dir}" dagster_project_dir = tmp_path.joinpath(project_name) result = runner.invoke( @@ -99,6 +115,7 @@ def test_project_scaffold_command_with_runtime_manifest( project_name, "--dbt-project-dir", os.fspath(dbt_project_dir), + *(["--use-dbt-project-package-data-dir"] if use_dbt_project_package_data_dir else []), ], ) @@ -111,6 +128,13 @@ def test_project_scaffold_command_with_runtime_manifest( assert not dbt_project_dir.joinpath("target", "manifest.json").exists() assert "dbt-duckdb" in dagster_project_dir.joinpath("setup.py").read_text() + if use_dbt_project_package_data_dir: + dbt_project_dir = dagster_project_dir.joinpath("dbt-project") + shutil.copytree( + src=test_dagster_metadata_dbt_project_path, + dst=dbt_project_dir, + ) + monkeypatch.chdir(tmp_path) sys.path.append(os.fspath(tmp_path))