From e2e273b7d17260654db7ca3d9ac13bba2e3a7690 Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Fri, 12 Jul 2024 14:13:59 -0400 Subject: [PATCH 1/3] add prepare_if_dev --- hooli_data_eng/project.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooli_data_eng/project.py b/hooli_data_eng/project.py index 8e32a0f..e3732bc 100644 --- a/hooli_data_eng/project.py +++ b/hooli_data_eng/project.py @@ -19,3 +19,4 @@ def get_env(): state_path="target/slim_ci", target=get_env(), ) +dbt_project.prepare_if_dev() \ No newline at end of file From 69b5b1ebad17342423b66622787e729a54df9fc1 Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Fri, 12 Jul 2024 14:14:11 -0400 Subject: [PATCH 2/3] fix typo --- dbt_project/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project/dbt_project.yml b/dbt_project/dbt_project.yml index 3e52a17..83acc3a 100644 --- a/dbt_project/dbt_project.yml +++ b/dbt_project/dbt_project.yml @@ -24,7 +24,7 @@ models: +group: Programmers CLEANED: +group: Programmers - +tags: "all_cleanred_models" + +tags: "all_cleaned_models" FORECASTING: +group: Stakeholders MARKETING: From 31685c1da8c766cafc8f299e691bc58927eb77f4 Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Fri, 12 Jul 2024 14:59:21 -0400 Subject: [PATCH 3/3] use DbtCliResousrce consistently --- hooli_data_eng/resources/__init__.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/hooli_data_eng/resources/__init__.py b/hooli_data_eng/resources/__init__.py index 77c09db..bab6367 100644 --- a/hooli_data_eng/resources/__init__.py +++ b/hooli_data_eng/resources/__init__.py @@ -40,8 +40,6 @@ # # The production deployment on Dagster Cloud uses production Snowflake # and S3 resources - - client = mock.MagicMock() if get_env() == "PROD": @@ -67,7 +65,7 @@ "output_notebook_io_manager": ConfigurableLocalOutputNotebookIOManager(), "api": RawDataAPI.configure_at_launch(), "s3": ResourceDefinition.none_resource(), - "dbt": DbtCliClientResource(project_dir=DBT_PROJECT_DIR, target="LOCAL"), + "dbt": DbtCliResource(project_dir=dbt_project, target="LOCAL"), "dbt2": DbtCliResource(project_dir=dbt_project, target="LOCAL"), "pyspark": pyspark_resource, "step_launcher": ResourceDefinition.none_resource(), @@ -92,7 +90,7 @@ ), "output_notebook_io_manager": ConfigurableLocalOutputNotebookIOManager(), "api": RawDataAPI.configure_at_launch(), - "dbt": DbtCliClientResource(project_dir=DBT_PROJECT_DIR, target="BRANCH"), + "dbt": DbtCliResource(project_dir=dbt_project, target="BRANCH"), "dbt2": DbtCliResource(project_dir=dbt_project, target="BRANCH"), "pyspark": pyspark_resource, "step_launcher": db_step_launcher, @@ -117,8 +115,8 @@ ), "output_notebook_io_manager": ConfigurableLocalOutputNotebookIOManager(), "api": RawDataAPI.configure_at_launch(), - "dbt": DbtCliClientResource(project_dir=DBT_PROJECT_DIR, target="PROD"), - "dbt2": DbtCliResource(project_dir=DBT_PROJECT_DIR, target="PROD"), + "dbt": DbtCliResource(project_dir=dbt_project, target="PROD"), + "dbt2": DbtCliResource(project_dir=dbt_project, target="PROD"), "pyspark": pyspark_resource, "step_launcher": db_step_launcher, "monitor_fs": s3FileSystem(region_name="us-west-2", s3_bucket="hooli-demo"),