From 829329c09df14fa8b3cc0de657558f508a9ffa5e Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Tue, 21 May 2024 10:46:13 -0400 Subject: [PATCH 1/3] add new fetch row_count to dbt_assets --- hooli_data_eng/assets/dbt_assets.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/hooli_data_eng/assets/dbt_assets.py b/hooli_data_eng/assets/dbt_assets.py index 740a9425..c5dc1429 100644 --- a/hooli_data_eng/assets/dbt_assets.py +++ b/hooli_data_eng/assets/dbt_assets.py @@ -115,7 +115,11 @@ def _process_partitioned_dbt_assets(context: OpExecutionContext, dbt: DbtCliReso # Emits an AssetObservation for each asset materialization, which is used to # identify the Snowflake credit consumption - yield from dbt_with_snowflake_insights(context, dbt_cli_task) + yield from dbt_with_snowflake_insights( + context=context, + dbt_cli_invocation=dbt_cli_task, + dagster_events=dbt_cli_task.stream().fetch_row_counts(), + ) # fetch run_results.json to log compiled SQL run_results_json = dbt_cli_task.get_artifact("run_results.json") @@ -163,7 +167,11 @@ def views_dbt_assets(context: OpExecutionContext, dbt2: DbtCliResource): # Emits an AssetObservation for each asset materialization, which is used to # identify the Snowflake credit consumption - yield from dbt_with_snowflake_insights(context, dbt_cli_task) + yield from dbt_with_snowflake_insights( + context=context, + dbt_cli_invocation=dbt_cli_task, + dagster_events=dbt_cli_task.stream().fetch_row_counts(), + ) # fetch run_results.json to log compiled SQL run_results_json = dbt_cli_task.get_artifact("run_results.json") @@ -190,7 +198,7 @@ def dbt_slim_ci(dbt2: DbtCliResource): dagster_dbt_translator=CustomDagsterDbtTranslator( DagsterDbtTranslatorSettings(enable_asset_checks=True) ), - ).stream() + ).stream().fetch_row_counts() # This job will be triggered by Pull Request and should only run new or changed dbt models From 0df1abb8f3462494702e0f3b9fea4df8ce7a536e Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Wed, 22 May 2024 15:35:54 -0400 Subject: [PATCH 2/3] remove dbt version pins --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e6942365..c5e9a57a 100644 --- a/setup.py +++ b/setup.py @@ -12,9 +12,9 @@ "pandas", "numpy", "scipy", - "dbt-core~=1.7.0", - "dbt-duckdb~=1.7.5", - "dbt-snowflake~=1.7.0", + "dbt-core", + "dbt-duckdb", + "dbt-snowflake", "dagster-duckdb", "dagster-aws", "dagster-duckdb-pandas", From 14d75afef4ebf66611ca147ab1053a7dfb434b8b Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Wed, 22 May 2024 15:56:10 -0400 Subject: [PATCH 3/3] remove pins from github actions --- .github/workflows/deploy-dagster-cloud.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-dagster-cloud.yml b/.github/workflows/deploy-dagster-cloud.yml index defdd21e..756f159c 100644 --- a/.github/workflows/deploy-dagster-cloud.yml +++ b/.github/workflows/deploy-dagster-cloud.yml @@ -78,7 +78,7 @@ jobs: if: steps.prerun.outputs.result != 'skip' run: | pip install pip --upgrade; - pip install dagster-dbt dagster-cloud dbt-core~=1.7.0 dbt-duckdb~=1.7.5 dbt-snowflake~=1.7.3 --upgrade --upgrade-strategy eager; + pip install dagster-dbt dagster-cloud dbt-core dbt-duckdb dbt-snowflake --upgrade --upgrade-strategy eager; make deps dagster-dbt project prepare-for-deployment --file hooli_data_eng/project.py dagster-cloud ci dagster-dbt project manage-state --file hooli_data_eng/project.py --source-deployment data-eng-prod