From 4c794228752d4711c98c49ebf8634f1942804358 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sat, 13 Jul 2024 12:15:08 -0600 Subject: [PATCH 01/25] Add SQL for source freshness results --- .../upload_source_executions.sql | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 macros/upload_individual_datasets/upload_source_executions.sql diff --git a/macros/upload_individual_datasets/upload_source_executions.sql b/macros/upload_individual_datasets/upload_source_executions.sql new file mode 100644 index 00000000..0b8a6cb2 --- /dev/null +++ b/macros/upload_individual_datasets/upload_source_executions.sql @@ -0,0 +1,147 @@ +{% macro upload_source_executions(sources) %} + {{ return(adapter.dispatch('get_source_freshness_dml_sql', 'dbt_artifacts')(freshness_results)) }} +{% endmacro %} + +{% macro default__get_source_executions_dml_sql(sources) -%} + {% if sources != [] and flags.source_freshness_run_project_hooks %} + + {% set source_execution_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(16) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(17) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(18) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(19) }} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(20)) }} + from values + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.thread_id }}', {# thread_id #} + '{{ source.status }}', {# status #} + + {% set compile_started_at = (source.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (source.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ source.execution_time }}, {# total_node_runtime #} + '{{ source.node.schema }}', {# schema #} + '{{ source.node.name }}', {# name #} + '{{ source.node.source_name }}', {# source_name #} + '{{ source.node.loaded_at_field }}', {# loaded_at_field #} + '{{ source.node.freshness.warn_after.count }}', {# warn_after_count #} + '{{ source.node.freshness.warn_after.period }}', {# warn_after_period #} + '{{ source.node.freshness.error_after.count }}', {# error_after_count #} + '{{ source.node.freshness.error_after.period }}', {# error_after_period #} + '{{ source.node.max_loaded_at }}', {# max_loaded_at #} + '{{ source.node.snapshotted_at }}', {# snapshotted_at #} + {{ source.node.age }}, {# age #} + '{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {% endfor %} + {% endset %} + {{ source_execution_values }} + {% else %} + {{ return ("") }} + {% endif %} +{% endmacro -%} + + +{% macro bigquery__get_source_executions_dml_sql(sources) -%} + {% if sources != [] and flags.source_freshness_run_project_hooks %} + + {% set source_execution_values %} + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.thread_id }}', {# thread_id #} + '{{ source.status }}', {# status #} + + {% set compile_started_at = (source.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (source.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ source.execution_time }}, {# total_node_runtime #} + '{{ source.node.schema }}', {# schema #} + '{{ source.node.name }}', {# name #} + '{{ source.node.source_name }}', {# source_name #} + '{{ source.node.loaded_at_field }}', {# loaded_at_field #} + '{{ source.node.freshness.warn_after.count }}', {# warn_after_count #} + '{{ source.node.freshness.warn_after.period }}', {# warn_after_period #} + '{{ source.node.freshness.error_after.count }}', {# error_after_count #} + '{{ source.node.freshness.error_after.period }}', {# error_after_period #} + '{{ source.node.max_loaded_at }}', {# max_loaded_at #} + '{{ source.node.snapshotted_at }}', {# snapshotted_at #} + {{ source.node.age }}, {# age #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {% endfor %} + {% endset %} + {{ source_execution_values }} + {% else %} + {{ return ("") }} + {% endif %} +{% endmacro -%} + + +{% macro postgres__get_source_executions_dml_sql(sources) -%} + {% if sources != [] and flags.source_freshness_run_project_hooks %} + + {% set source_execution_values %} + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.thread_id }}', {# thread_id #} + '{{ source.status }}', {# status #} + + {% set compile_started_at = (source.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (source.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ source.execution_time }}, {# total_node_runtime #} + '{{ source.node.schema }}', {# schema #} + '{{ source.node.name }}', {# name #} + '{{ source.node.source_name }}', {# source_name #} + '{{ source.node.loaded_at_field }}', {# loaded_at_field #} + '{{ source.node.freshness.warn_after.count }}', {# warn_after_count #} + '{{ source.node.freshness.warn_after.period }}', {# warn_after_period #} + '{{ source.node.freshness.error_after.count }}', {# error_after_count #} + '{{ source.node.freshness.error_after.period }}', {# error_after_period #} + '{{ source.node.max_loaded_at }}', {# max_loaded_at #} + '{{ source.node.snapshotted_at }}', {# snapshotted_at #} + {{ source.node.age }}, {# age #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {% endfor %} + {% endset %} + {{ source_execution_values }} + {% else %} + {{ return ("") }} + {% endif %} +{% endmacro -%} From b07abcae045115948be817fab7f647e245abd42d Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sat, 13 Jul 2024 12:15:32 -0600 Subject: [PATCH 02/25] Include source_executions when looping through --- macros/upload_results/get_dataset_content.sql | 2 +- macros/upload_results/get_table_content_values.sql | 2 ++ macros/upload_results/upload_results.sql | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/macros/upload_results/get_dataset_content.sql b/macros/upload_results/get_dataset_content.sql index b34e39cd..e318c69d 100644 --- a/macros/upload_results/get_dataset_content.sql +++ b/macros/upload_results/get_dataset_content.sql @@ -1,6 +1,6 @@ {% macro get_dataset_content(dataset) %} - {% if dataset in ['model_executions', 'seed_executions', 'test_executions', 'snapshot_executions'] %} + {% if dataset in ['model_executions', 'seed_executions', 'test_executions', 'snapshot_executions', 'source_executions'] %} {# Executions make use of the results object #} {% set objects = results | selectattr("node.resource_type", "equalto", dataset.split("_")[0]) | list %} {% elif dataset in ['seeds', 'snapshots', 'tests', 'models'] %} diff --git a/macros/upload_results/get_table_content_values.sql b/macros/upload_results/get_table_content_values.sql index 277899c5..6b459341 100644 --- a/macros/upload_results/get_table_content_values.sql +++ b/macros/upload_results/get_table_content_values.sql @@ -10,6 +10,8 @@ {% set content = dbt_artifacts.upload_test_executions(objects_to_upload) %} {% elif dataset == 'snapshot_executions' %} {% set content = dbt_artifacts.upload_snapshot_executions(objects_to_upload) %} + {% elif dataset == 'source_executions' %} + {% set content = dbt_artifacts.upload_source_executions(objects_to_upload) %} {% elif dataset == 'exposures' %} {% set content = dbt_artifacts.upload_exposures(objects_to_upload) %} {% elif dataset == 'models' %} diff --git a/macros/upload_results/upload_results.sql b/macros/upload_results/upload_results.sql index 114a667d..6e06121e 100644 --- a/macros/upload_results/upload_results.sql +++ b/macros/upload_results/upload_results.sql @@ -7,7 +7,7 @@ {% set datasets_to_load = ['exposures', 'seeds', 'snapshots', 'invocations', 'sources', 'tests', 'models'] %} {% if results != [] %} {# When executing, and results are available, then upload the results #} - {% set datasets_to_load = ['model_executions', 'seed_executions', 'test_executions', 'snapshot_executions'] + datasets_to_load %} + {% set datasets_to_load = ['model_executions', 'seed_executions', 'test_executions', 'snapshot_executions', 'source_executions'] + datasets_to_load %} {% endif %} {# Upload each data set in turn #} From 727039687eedbb9a6c50b8d805317e2a15588474 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sat, 13 Jul 2024 12:19:02 -0600 Subject: [PATCH 03/25] Fix for wrong macro name --- macros/upload_individual_datasets/upload_source_executions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_individual_datasets/upload_source_executions.sql b/macros/upload_individual_datasets/upload_source_executions.sql index 0b8a6cb2..6d637a3c 100644 --- a/macros/upload_individual_datasets/upload_source_executions.sql +++ b/macros/upload_individual_datasets/upload_source_executions.sql @@ -1,5 +1,5 @@ {% macro upload_source_executions(sources) %} - {{ return(adapter.dispatch('get_source_freshness_dml_sql', 'dbt_artifacts')(freshness_results)) }} + {{ return(adapter.dispatch('get_source_executions_dml_sql', 'dbt_artifacts')(freshness_results)) }} {% endmacro %} {% macro default__get_source_executions_dml_sql(sources) -%} From 23769ed75cc3586ad85a1d4de4a700f5df7b2647 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sat, 13 Jul 2024 12:29:28 -0600 Subject: [PATCH 04/25] Add column names for source executions --- .../upload_results/get_column_name_lists.sql | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/macros/upload_results/get_column_name_lists.sql b/macros/upload_results/get_column_name_lists.sql index 7911e866..083cf13d 100644 --- a/macros/upload_results/get_column_name_lists.sql +++ b/macros/upload_results/get_column_name_lists.sql @@ -221,6 +221,31 @@ all_results ) + {% elif dataset == 'source_executions' %} + + ( + command_invocation_id, + node_id, + run_started_at, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + schema, + name, + source_name, + loaded_at_field, + warn_after_count, + warn_after_period, + error_after_count, + error_after_period, + max_loaded_at, + snapshotted_at, + age, + adapter_response + ) + {% else %} /* No column list available */ From baac8a525a325504a60a1e630368dbc012bfe90b Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sat, 13 Jul 2024 12:52:01 -0600 Subject: [PATCH 05/25] Fix error passing in old argument name --- .../upload_source_executions.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/macros/upload_individual_datasets/upload_source_executions.sql b/macros/upload_individual_datasets/upload_source_executions.sql index 6d637a3c..34f5e49f 100644 --- a/macros/upload_individual_datasets/upload_source_executions.sql +++ b/macros/upload_individual_datasets/upload_source_executions.sql @@ -1,9 +1,9 @@ {% macro upload_source_executions(sources) %} - {{ return(adapter.dispatch('get_source_executions_dml_sql', 'dbt_artifacts')(freshness_results)) }} + {{ return(adapter.dispatch('get_source_executions_dml_sql', 'dbt_artifacts')(sources)) }} {% endmacro %} {% macro default__get_source_executions_dml_sql(sources) -%} - {% if sources != [] and flags.source_freshness_run_project_hooks %} + {% if sources != [] %} {% set source_execution_values %} select @@ -22,7 +22,7 @@ {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(16) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(16) }},4 {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(17) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(18) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(19) }} @@ -66,7 +66,7 @@ {% macro bigquery__get_source_executions_dml_sql(sources) -%} - {% if sources != [] and flags.source_freshness_run_project_hooks %} + {% if sources != [] %} {% set source_execution_values %} {% for source in sources -%} @@ -107,7 +107,7 @@ {% macro postgres__get_source_executions_dml_sql(sources) -%} - {% if sources != [] and flags.source_freshness_run_project_hooks %} + {% if sources != [] %} {% set source_execution_values %} {% for source in sources -%} From a58322657375ff71158919c9b66c9b6a3b8e5203 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sat, 13 Jul 2024 12:54:11 -0600 Subject: [PATCH 06/25] Add flag to integration test project --- integration_test_project/dbt_project.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index dba89a50..232636d3 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -30,5 +30,8 @@ models: seeds: +quote_columns: false +flags: + source_freshness_run_project_hooks: true + on-run-end: - "{{ dbt_artifacts.upload_results(results) }}" From 24b52728c7a28a133239be6ca538cd437f5d1643 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sat, 13 Jul 2024 13:18:07 -0600 Subject: [PATCH 07/25] Oof --- macros/upload_individual_datasets/upload_source_executions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_individual_datasets/upload_source_executions.sql b/macros/upload_individual_datasets/upload_source_executions.sql index 34f5e49f..0509b417 100644 --- a/macros/upload_individual_datasets/upload_source_executions.sql +++ b/macros/upload_individual_datasets/upload_source_executions.sql @@ -22,7 +22,7 @@ {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(16) }},4 + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(16) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(17) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(18) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(19) }} From 114e77235a0be117de675bdecdb1dd853315032d Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 09:37:21 -0600 Subject: [PATCH 08/25] Count values shouldn't be strings --- .../upload_individual_datasets/upload_source_executions.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/upload_individual_datasets/upload_source_executions.sql b/macros/upload_individual_datasets/upload_source_executions.sql index 0509b417..e4fc3335 100644 --- a/macros/upload_individual_datasets/upload_source_executions.sql +++ b/macros/upload_individual_datasets/upload_source_executions.sql @@ -46,9 +46,9 @@ '{{ source.node.name }}', {# name #} '{{ source.node.source_name }}', {# source_name #} '{{ source.node.loaded_at_field }}', {# loaded_at_field #} - '{{ source.node.freshness.warn_after.count }}', {# warn_after_count #} + {{ source.node.freshness.warn_after.count }}, {# warn_after_count #} '{{ source.node.freshness.warn_after.period }}', {# warn_after_period #} - '{{ source.node.freshness.error_after.count }}', {# error_after_count #} + {{ source.node.freshness.error_after.count }}, {# error_after_count #} '{{ source.node.freshness.error_after.period }}', {# error_after_period #} '{{ source.node.max_loaded_at }}', {# max_loaded_at #} '{{ source.node.snapshotted_at }}', {# snapshotted_at #} From c93bbfa53e617c17bd14f6a16d2d2fd09ff33d06 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 09:37:57 -0600 Subject: [PATCH 09/25] Fix attributes not on node --- .../upload_individual_datasets/upload_source_executions.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/macros/upload_individual_datasets/upload_source_executions.sql b/macros/upload_individual_datasets/upload_source_executions.sql index e4fc3335..61fe7d1e 100644 --- a/macros/upload_individual_datasets/upload_source_executions.sql +++ b/macros/upload_individual_datasets/upload_source_executions.sql @@ -50,9 +50,9 @@ '{{ source.node.freshness.warn_after.period }}', {# warn_after_period #} {{ source.node.freshness.error_after.count }}, {# error_after_count #} '{{ source.node.freshness.error_after.period }}', {# error_after_period #} - '{{ source.node.max_loaded_at }}', {# max_loaded_at #} - '{{ source.node.snapshotted_at }}', {# snapshotted_at #} - {{ source.node.age }}, {# age #} + '{{ source.max_loaded_at }}', {# max_loaded_at #} + '{{ source.snapshotted_at }}', {# snapshotted_at #} + {{ source.age }}, {# age #} '{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} ) {%- if not loop.last %},{%- endif %} From bf5312eebc816c5e028eda9de5f15beb1e339aa6 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 09:42:07 -0600 Subject: [PATCH 10/25] Copy/paste fun --- macros/upload_individual_datasets/upload_source_executions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_individual_datasets/upload_source_executions.sql b/macros/upload_individual_datasets/upload_source_executions.sql index 61fe7d1e..b20ee143 100644 --- a/macros/upload_individual_datasets/upload_source_executions.sql +++ b/macros/upload_individual_datasets/upload_source_executions.sql @@ -53,7 +53,7 @@ '{{ source.max_loaded_at }}', {# max_loaded_at #} '{{ source.snapshotted_at }}', {# snapshotted_at #} {{ source.age }}, {# age #} - '{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} + '{{ tojson(source.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {% endfor %} From 43285e2757c6d87a185f7afc06e1ebb3024e673f Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 09:52:46 -0600 Subject: [PATCH 11/25] Missing comma --- .../upload_source_executions.sql | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/macros/upload_individual_datasets/upload_source_executions.sql b/macros/upload_individual_datasets/upload_source_executions.sql index b20ee143..bf0a2b71 100644 --- a/macros/upload_individual_datasets/upload_source_executions.sql +++ b/macros/upload_individual_datasets/upload_source_executions.sql @@ -106,6 +106,68 @@ {% endmacro -%} +{% macro snowflake__get_source_executions_dml_sql(sources) -%} + {% if sources != [] %} + + {% set source_execution_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(16) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(17) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(18) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(19) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(20)) }} + from values + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.thread_id }}', {# thread_id #} + '{{ source.status }}', {# status #} + + {% set compile_started_at = (source.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (source.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ source.execution_time }}, {# total_node_runtime #} + '{{ source.node.schema }}', {# schema #} + '{{ source.node.name }}', {# name #} + '{{ source.node.source_name }}', {# source_name #} + '{{ source.node.loaded_at_field }}', {# loaded_at_field #} + {{ source.node.freshness.warn_after.count }}, {# warn_after_count #} + '{{ source.node.freshness.warn_after.period }}', {# warn_after_period #} + {{ source.node.freshness.error_after.count }}, {# error_after_count #} + '{{ source.node.freshness.error_after.period }}', {# error_after_period #} + '{{ source.max_loaded_at }}', {# max_loaded_at #} + '{{ source.snapshotted_at }}', {# snapshotted_at #} + {{ source.age }}, {# age #} + '{{ tojson(source.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {% endfor %} + {% endset %} + {{ source_execution_values }} + {% else %} + {{ return ("") }} + {% endif %} +{% endmacro -%} + {% macro postgres__get_source_executions_dml_sql(sources) -%} {% if sources != [] %} From 524063125996539bd1fa8805961014512b7e15ff Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 10:30:49 -0600 Subject: [PATCH 12/25] Add missing comma --- macros/upload_individual_datasets/upload_source_executions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_individual_datasets/upload_source_executions.sql b/macros/upload_individual_datasets/upload_source_executions.sql index bf0a2b71..636684e7 100644 --- a/macros/upload_individual_datasets/upload_source_executions.sql +++ b/macros/upload_individual_datasets/upload_source_executions.sql @@ -25,7 +25,7 @@ {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(16) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(17) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(18) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(19) }} + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(19) }}, {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(20)) }} from values {% for source in sources -%} From 68b1288de6a40fde9ad32a8618649d67054b7b8c Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 11:32:10 -0600 Subject: [PATCH 13/25] Add source executions model and yml --- models/sources/source_executions.sql | 29 ++++++++++++++++++ models/sources/source_executions.yml | 46 ++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 models/sources/source_executions.sql create mode 100644 models/sources/source_executions.yml diff --git a/models/sources/source_executions.sql b/models/sources/source_executions.sql new file mode 100644 index 00000000..b42cd47e --- /dev/null +++ b/models/sources/source_executions.sql @@ -0,0 +1,29 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as source_name, + cast(null as {{ type_timestamp() }}) as loaded_at_field, + cast(null as {{ type_int() }}) as warn_after_count, + cast(null as {{ type_string() }}) as warn_after_period, + cast(null as {{ type_int() }}) as error_after_count, + cast(null as {{ type_string() }}) as error_after_period, + cast(null as {{ type_timestamp() }}) as max_loaded_at, + cast(null as {{ type_timestamp() }}) as snapshotted_at, + cast(null as {{ type_float() }}) as age, + cast(null as {{ type_json() }}) as adapter_response +from dummy_cte +where 1 = 0 diff --git a/models/sources/source_executions.yml b/models/sources/source_executions.yml new file mode 100644 index 00000000..347bad55 --- /dev/null +++ b/models/sources/source_executions.yml @@ -0,0 +1,46 @@ +version: 2 + +models: + - name: source_executions + description: Base model for data about model executions. One row per model execution. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: thread_id + description: '{{ doc("thread_id") }}' + - name: status + description: '{{ doc("status") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' + - name: query_completed_at + description: '{{ doc("query_completed_at") }}' + - name: total_node_runtime + description: '{{ doc("total_node_runtime") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: name + description: '{{ doc("name") }}' + - name: source_name + description: '{{ doc("source_name") }}' + - name: loaded_at_field + description: '{{ doc("loaded_at_field") }}' + - name: warn_after_count + description: '{{ doc("warn_after_count") }}' + - name: warn_after_period + description: '{{ doc("warn_after_period") }}' + - name: error_after_count + description: '{{ doc("error_after_count") }}' + - name: error_after_period + description: '{{ doc("error_after_period") }}' + - name: max_loaded_at + description: '{{ doc("max_loaded_at") }}' + - name: snapshotted_at + description: '{{ doc("snapshotted_at") }}' + - name: age + description: '{{ doc("age") }}' + - name: adapter_response + description: '{{ doc("adapter_response") }}' From 8f7a167384cffde0a18d1e133afc37ebdc3df4c8 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 11:32:41 -0600 Subject: [PATCH 14/25] Add missing docs blocks for new source_executions model/yml --- models/docs.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/models/docs.md b/models/docs.md index 64b420cb..c87f046a 100644 --- a/models/docs.md +++ b/models/docs.md @@ -412,3 +412,51 @@ Response provided by the adapter as JSON. All results as a JSON blob {% enddocs %} + +{% docs source_name %} + +The name of the source + +{% enddocs %} + +{% docs warn_after_count %} + +Positive integer indicating a threshold to warn (used alongside warn_after_period) + +{% enddocs %} + +{% docs warn_after_period %} + +Used alongside warn_after_count to indicate number of periods (minutes, hours, days) for a warn threshold + +{% enddocs %} + +{% docs error_after_count %} + +Positive integer indicating a threshold to error (used alongside error_after_period) + +{% enddocs %} + +{% docs error_after_period %} + +Used alongside error_after_count to indicate number of periods (minutes, hours, days) for a error threshold + +{% enddocs %} + +{% docs max_loaded_at %} + +Max value of loaded_at_field timestamp in the source table when queried + +{% enddocs %} + +{% docs snapshotted_at %} + +Current timestamp when querying + +{% enddocs %} + +{% docs age %} + +Interval between max_loaded_at and snapshotted_at, calculated in python to handle timezone complexity + +{% enddocs %} \ No newline at end of file From 9f115e755114c90b1f53c2cd9a0c5c5fc8f36828 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 11:34:21 -0600 Subject: [PATCH 15/25] Add staging model for source executions --- models/staging/stg_dbt__source_executions.sql | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 models/staging/stg_dbt__source_executions.sql diff --git a/models/staging/stg_dbt__source_executions.sql b/models/staging/stg_dbt__source_executions.sql new file mode 100644 index 00000000..988be3b7 --- /dev/null +++ b/models/staging/stg_dbt__source_executions.sql @@ -0,0 +1,36 @@ +with base as ( + + select * + from {{ ref('model_executions') }} + +), + +enhanced as ( + + select + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id, + command_invocation_id, + node_id, + run_started_at, + {{ split_part('thread_id', "'-'", 2) }} as thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + schema, -- noqa + name, + source_name, + loaded_at_field, + warn_after_count, + warn_after_period, + error_after_count, + error_after_period, + max_loaded_at, + snapshotted_at, + age, + adapter_response + from base + +) + +select * from enhanced From 3a2b0c513a440bcd23f29da6073c78bb460ea3a7 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 11:34:56 -0600 Subject: [PATCH 16/25] Fix id name --- models/staging/stg_dbt__source_executions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/stg_dbt__source_executions.sql b/models/staging/stg_dbt__source_executions.sql index 988be3b7..8b3056b2 100644 --- a/models/staging/stg_dbt__source_executions.sql +++ b/models/staging/stg_dbt__source_executions.sql @@ -8,7 +8,7 @@ with base as ( enhanced as ( select - {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id, + {{ dbt_artifacts.generate_surrogate_key(['command_invocation_id', 'node_id']) }} as source_execution_id, command_invocation_id, node_id, run_started_at, From d85d62de0bf7f58b29d156f7f804ba2e41d06bc7 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 11:39:30 -0600 Subject: [PATCH 17/25] Add staging / fact models and yml for source executions --- models/fct_dbt__source_executions.sql | 36 +++++++++++++++ models/fct_dbt__source_executions.yml | 46 +++++++++++++++++++ models/staging/stg_dbt__source_executions.yml | 46 +++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 models/fct_dbt__source_executions.sql create mode 100644 models/fct_dbt__source_executions.yml create mode 100644 models/staging/stg_dbt__source_executions.yml diff --git a/models/fct_dbt__source_executions.sql b/models/fct_dbt__source_executions.sql new file mode 100644 index 00000000..33b797f6 --- /dev/null +++ b/models/fct_dbt__source_executions.sql @@ -0,0 +1,36 @@ +with base as ( + + select * + from {{ ref('stg_dbt__source_executions') }} + +), + +source_executions as ( + + select + source_execution_id, + command_invocation_id, + node_id, + run_started_at, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + schema, -- noqa + name, + source_name, + loaded_at_field, + warn_after_count, + warn_after_period, + error_after_count, + error_after_period, + max_loaded_at, + snapshotted_at, + age, + adapter_response + from base + +) + +select * from source_executions diff --git a/models/fct_dbt__source_executions.yml b/models/fct_dbt__source_executions.yml new file mode 100644 index 00000000..36e4e612 --- /dev/null +++ b/models/fct_dbt__source_executions.yml @@ -0,0 +1,46 @@ +version: 2 + +models: + - name: fct_dbt__source_executions + description: Fact model for data about source freshness executions. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: thread_id + description: '{{ doc("thread_id") }}' + - name: status + description: '{{ doc("status") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' + - name: query_completed_at + description: '{{ doc("query_completed_at") }}' + - name: total_node_runtime + description: '{{ doc("total_node_runtime") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: name + description: '{{ doc("name") }}' + - name: source_name + description: '{{ doc("source_name") }}' + - name: loaded_at_field + description: '{{ doc("loaded_at_field") }}' + - name: warn_after_count + description: '{{ doc("warn_after_count") }}' + - name: warn_after_period + description: '{{ doc("warn_after_period") }}' + - name: error_after_count + description: '{{ doc("error_after_count") }}' + - name: error_after_period + description: '{{ doc("error_after_period") }}' + - name: max_loaded_at + description: '{{ doc("max_loaded_at") }}' + - name: snapshotted_at + description: '{{ doc("snapshotted_at") }}' + - name: age + description: '{{ doc("age") }}' + - name: adapter_response + description: '{{ doc("adapter_response") }}' diff --git a/models/staging/stg_dbt__source_executions.yml b/models/staging/stg_dbt__source_executions.yml new file mode 100644 index 00000000..6e6ef87d --- /dev/null +++ b/models/staging/stg_dbt__source_executions.yml @@ -0,0 +1,46 @@ +version: 2 + +models: + - name: stg_dbt__source_executions + description: Staging model for data about source freshness executions. One row per source freshness execution. + columns: + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: node_id + description: '{{ doc("node_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: thread_id + description: '{{ doc("thread_id") }}' + - name: status + description: '{{ doc("status") }}' + - name: compile_started_at + description: '{{ doc("compile_started_at") }}' + - name: query_completed_at + description: '{{ doc("query_completed_at") }}' + - name: total_node_runtime + description: '{{ doc("total_node_runtime") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: name + description: '{{ doc("name") }}' + - name: source_name + description: '{{ doc("source_name") }}' + - name: loaded_at_field + description: '{{ doc("loaded_at_field") }}' + - name: warn_after_count + description: '{{ doc("warn_after_count") }}' + - name: warn_after_period + description: '{{ doc("warn_after_period") }}' + - name: error_after_count + description: '{{ doc("error_after_count") }}' + - name: error_after_period + description: '{{ doc("error_after_period") }}' + - name: max_loaded_at + description: '{{ doc("max_loaded_at") }}' + - name: snapshotted_at + description: '{{ doc("snapshotted_at") }}' + - name: age + description: '{{ doc("age") }}' + - name: adapter_response + description: '{{ doc("adapter_response") }}' From 1a511bc30bf7dc93c49afdcdab7aa773a456109e Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 11:40:41 -0600 Subject: [PATCH 18/25] Remove unnecessary doc block --- models/docs.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/models/docs.md b/models/docs.md index c87f046a..b884e896 100644 --- a/models/docs.md +++ b/models/docs.md @@ -413,12 +413,6 @@ All results as a JSON blob {% enddocs %} -{% docs source_name %} - -The name of the source - -{% enddocs %} - {% docs warn_after_count %} Positive integer indicating a threshold to warn (used alongside warn_after_period) From 5b61f19d03a4cbf65c156f5125deb8ccef3588ba Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 11:44:35 -0600 Subject: [PATCH 19/25] Fix ref --- models/staging/stg_dbt__source_executions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/stg_dbt__source_executions.sql b/models/staging/stg_dbt__source_executions.sql index 8b3056b2..425fc529 100644 --- a/models/staging/stg_dbt__source_executions.sql +++ b/models/staging/stg_dbt__source_executions.sql @@ -1,7 +1,7 @@ with base as ( select * - from {{ ref('model_executions') }} + from {{ ref('source_executions') }} ), From 1e28b7cd4d3f08dfaa551509b0d6d5c27f3454ac Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 11:46:10 -0600 Subject: [PATCH 20/25] Should be a string --- models/sources/source_executions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/sources/source_executions.sql b/models/sources/source_executions.sql index b42cd47e..fd3fc25f 100644 --- a/models/sources/source_executions.sql +++ b/models/sources/source_executions.sql @@ -16,7 +16,7 @@ select cast(null as {{ type_string() }}) as schema, cast(null as {{ type_string() }}) as name, cast(null as {{ type_string() }}) as source_name, - cast(null as {{ type_timestamp() }}) as loaded_at_field, + cast(null as {{ type_string() }}) as loaded_at_field, cast(null as {{ type_int() }}) as warn_after_count, cast(null as {{ type_string() }}) as warn_after_period, cast(null as {{ type_int() }}) as error_after_count, From 4a330a7e6a2a6afc543b07ce6505761b9115baba Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 11:50:28 -0600 Subject: [PATCH 21/25] Fix other adapter dml --- .../upload_source_executions.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/macros/upload_individual_datasets/upload_source_executions.sql b/macros/upload_individual_datasets/upload_source_executions.sql index 636684e7..17782a76 100644 --- a/macros/upload_individual_datasets/upload_source_executions.sql +++ b/macros/upload_individual_datasets/upload_source_executions.sql @@ -91,9 +91,9 @@ '{{ source.node.freshness.warn_after.period }}', {# warn_after_period #} '{{ source.node.freshness.error_after.count }}', {# error_after_count #} '{{ source.node.freshness.error_after.period }}', {# error_after_period #} - '{{ source.node.max_loaded_at }}', {# max_loaded_at #} - '{{ source.node.snapshotted_at }}', {# snapshotted_at #} - {{ source.node.age }}, {# age #} + '{{ source.max_loaded_at }}', {# max_loaded_at #} + '{{ source.snapshotted_at }}', {# snapshotted_at #} + {{ source.age }}, {# age #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# adapter_response #} ) {%- if not loop.last %},{%- endif %} @@ -194,9 +194,9 @@ '{{ source.node.freshness.warn_after.period }}', {# warn_after_period #} '{{ source.node.freshness.error_after.count }}', {# error_after_count #} '{{ source.node.freshness.error_after.period }}', {# error_after_period #} - '{{ source.node.max_loaded_at }}', {# max_loaded_at #} - '{{ source.node.snapshotted_at }}', {# snapshotted_at #} - {{ source.node.age }}, {# age #} + '{{ source.max_loaded_at }}', {# max_loaded_at #} + '{{ source.snapshotted_at }}', {# snapshotted_at #} + {{ source.age }}, {# age #} $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} ) {%- if not loop.last %},{%- endif %} From 53f49f49e8c55eed24582c5e03a7d0d84ead81b1 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 11:53:12 -0600 Subject: [PATCH 22/25] Remove full refresh column from 'source' --- models/sources/source_executions.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/models/sources/source_executions.sql b/models/sources/source_executions.sql index fd3fc25f..f48aef0a 100644 --- a/models/sources/source_executions.sql +++ b/models/sources/source_executions.sql @@ -7,7 +7,6 @@ select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_boolean() }}) as was_full_refresh, cast(null as {{ type_string() }}) as thread_id, cast(null as {{ type_string() }}) as status, cast(null as {{ type_timestamp() }}) as compile_started_at, From df041c62a884c9ad54a17153f0b7c83fa2efb89c Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 15:06:00 -0600 Subject: [PATCH 23/25] Add source freshness command to testing suite --- tox.ini | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tox.ini b/tox.ini index 10880752..a795c24a 100644 --- a/tox.ini +++ b/tox.ini @@ -124,6 +124,7 @@ deps = dbt-snowflake~=1.8.0 commands = dbt clean dbt deps + dbt source freshness --target snowflake dbt build --target snowflake @@ -133,6 +134,7 @@ deps = dbt-snowflake~=1.3.0 commands = dbt clean dbt deps + dbt source freshness --target snowflake dbt build --target snowflake [testenv:integration_snowflake_1_4_0] @@ -141,6 +143,7 @@ deps = dbt-snowflake~=1.4.0 commands = dbt clean dbt deps + dbt source freshness --target snowflake dbt build --target snowflake [testenv:integration_snowflake_1_5_0] @@ -149,6 +152,7 @@ deps = dbt-snowflake~=1.5.0 commands = dbt clean dbt deps + dbt source freshness --target snowflake dbt build --target snowflake [testenv:integration_snowflake_1_6_0] @@ -157,6 +161,7 @@ deps = dbt-snowflake~=1.6.0 commands = dbt clean dbt deps + dbt source freshness --target snowflake dbt build --target snowflake [testenv:integration_snowflake_1_7_0] @@ -165,6 +170,7 @@ deps = dbt-snowflake~=1.7.0 commands = dbt clean dbt deps + dbt source freshness --target snowflake dbt build --target snowflake [testenv:integration_snowflake_1_8_0] @@ -173,6 +179,7 @@ deps = dbt-snowflake~=1.8.0 commands = dbt clean dbt deps + dbt source freshness --target snowflake dbt build --target snowflake # Databricks integration tests @@ -182,6 +189,7 @@ deps = dbt-databricks~=1.8.0 commands = dbt clean dbt deps + dbt source freshness --target databricks dbt build --target databricks [testenv:integration_databricks_1_3_0] @@ -190,6 +198,7 @@ deps = dbt-databricks~=1.3.0 commands = dbt clean dbt deps + dbt source freshness --target databricks dbt build --target databricks [testenv:integration_databricks_1_4_0] @@ -198,6 +207,7 @@ deps = dbt-databricks~=1.4.0 commands = dbt clean dbt deps + dbt source freshness --target databricks dbt build --target databricks [testenv:integration_databricks_1_5_0] @@ -206,6 +216,7 @@ deps = dbt-databricks~=1.5.0 commands = dbt clean dbt deps + dbt source freshness --target databricks dbt build --target databricks [testenv:integration_databricks_1_6_0] @@ -214,6 +225,7 @@ deps = dbt-databricks~=1.6.0 commands = dbt clean dbt deps + dbt source freshness --target databricks dbt build --target databricks [testenv:integration_databricks_1_7_0] @@ -222,6 +234,7 @@ deps = dbt-databricks~=1.7.0 commands = dbt clean dbt deps + dbt source freshness --target databricks dbt build --target databricks [testenv:integration_databricks_1_8_0] @@ -230,6 +243,7 @@ deps = dbt-databricks~=1.8.0 commands = dbt clean dbt deps + dbt source freshness --target databricks dbt build --target databricks # Bigquery integration tests @@ -239,6 +253,7 @@ deps = dbt-bigquery~=1.8.0 commands = dbt clean dbt deps + dbt source freshness --target bigquery --vars '"my_var": "my value"' dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_3_0] @@ -247,6 +262,7 @@ deps = dbt-bigquery~=1.3.0 commands = dbt clean dbt deps + dbt source freshness --target bigquery --vars '"my_var": "my value"' dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_4_0] @@ -255,6 +271,7 @@ deps = dbt-bigquery~=1.4.0 commands = dbt clean dbt deps + dbt source freshness --target bigquery --vars '"my_var": "my value"' dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_5_0] @@ -263,6 +280,7 @@ deps = dbt-bigquery~=1.5.0 commands = dbt clean dbt deps + dbt source freshness --target bigquery --vars '"my_var": "my value"' dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_6_0] @@ -271,6 +289,7 @@ deps = dbt-bigquery~=1.6.0 commands = dbt clean dbt deps + dbt source freshness --target bigquery --vars '"my_var": "my value"' dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_7_0] @@ -279,6 +298,7 @@ deps = dbt-bigquery~=1.7.0 commands = dbt clean dbt deps + dbt source freshness --target bigquery --vars '"my_var": "my value"' dbt build --target bigquery --vars '"my_var": "my value"' [testenv:integration_bigquery_1_8_0] @@ -287,6 +307,7 @@ deps = dbt-bigquery~=1.8.0 commands = dbt clean dbt deps + dbt source freshness --target bigquery --vars '"my_var": "my value"' dbt build --target bigquery --vars '"my_var": "my value"' # Spark integration test (disabled) @@ -296,6 +317,7 @@ deps = dbt-spark[ODBC]~=1.4.0 commands = dbt clean dbt deps + dbt source freshness --target spark dbt build --exclude snapshot --target spark [testenv:integration_postgres] @@ -304,6 +326,7 @@ deps = dbt-postgres~=1.8.0 commands = dbt clean dbt deps + dbt source freshness --target postgres dbt build --target postgres [testenv:integration_postgres_1_3_0] @@ -312,6 +335,7 @@ deps = dbt-postgres~=1.3.0 commands = dbt clean dbt deps + dbt source freshness --target postgres dbt build --target postgres [testenv:integration_postgres_1_4_0] @@ -320,6 +344,7 @@ deps = dbt-postgres~=1.4.0 commands = dbt clean dbt deps + dbt source freshness --target postgres dbt build --target postgres [testenv:integration_postgres_1_5_0] @@ -328,6 +353,7 @@ deps = dbt-postgres~=1.5.0 commands = dbt clean dbt deps + dbt source freshness --target postgres dbt build --target postgres [testenv:integration_postgres_1_6_0] @@ -336,6 +362,7 @@ deps = dbt-postgres~=1.6.0 commands = dbt clean dbt deps + dbt source freshness --target postgres dbt build --target postgres [testenv:integration_postgres_1_7_0] @@ -344,6 +371,7 @@ deps = dbt-postgres~=1.7.0 commands = dbt clean dbt deps + dbt source freshness --target postgres dbt build --target postgres [testenv:integration_postgres_1_8_0] @@ -352,6 +380,7 @@ deps = dbt-postgres~=1.7.0 commands = dbt clean dbt deps + dbt source freshness --target postgres dbt build --target postgres From b00dfab435d9c7bac8683de6019de4e9530e6aaf Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Sun, 14 Jul 2024 15:12:31 -0600 Subject: [PATCH 24/25] Update README for instructions for opting in --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 166d708a..97138666 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ fct_dbt__model_executions fct_dbt__seed_executions fct_dbt__snapshot_executions fct_dbt__test_executions +fct_dbt__source_executions ``` See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/#!/overview) for documentation on each model. @@ -73,6 +74,13 @@ packages: dbt run --select dbt_artifacts ``` +5. If you want to include results from the `dbt source freshness` command, you'll need to add the following flag in your `dbt_project.yml`. This was recently added and made backwards-compatible to older core versions, so it's something that you have to opt into. + + ```yml + flags: + source_freshness_run_project_hooks: true + ``` + ### Notes on upgrading Due to the structure of the project, when additional fields are added, the package needs to be re-run to ensure the tables include the new field, or it will simply error on the hook. These changes will always be implemented within a new **minor** version, so make sure that the version you use in `packages.yml` reflects this. From 35dc474e288b5863eb91fd3e1d7f416b7bc80bbe Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Wed, 21 Aug 2024 16:14:52 +0000 Subject: [PATCH 25/25] Take config from profiles.yml and put in dbt_project.yml --- integration_test_project/dbt_project.yml | 2 ++ integration_test_project/profiles.yml | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index 232636d3..892c4c60 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -32,6 +32,8 @@ seeds: flags: source_freshness_run_project_hooks: true + send_anonymous_usage_stats: false + use_colors: true on-run-end: - "{{ dbt_artifacts.upload_results(results) }}" diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index b24ad80d..e611255e 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -1,10 +1,6 @@ # HEY! This file is used in the dbt-artifacts integrations tests with GitHub Actions. # You should __NEVER__ check credentials into version control. Thanks for reading :) -config: - send_anonymous_usage_stats: False - use_colors: True - dbt_artifacts: target: snowflake outputs: