From 9d4c3021a023e27b1837653e9c566d55451aa72e Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 26 Aug 2024 15:16:47 -0400 Subject: [PATCH 01/16] Update snapshots/helpers.sql and snapshots/snapshot_merge.sql to use config column_names --- .../materializations/snapshots/helpers.sql | 43 +++++++++++-------- .../snapshots/snapshot_merge.sql | 9 ++-- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index bb71974c..c218b8d9 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -35,6 +35,11 @@ {% endmacro %} {% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} + dbt_scd_id = config.get('dbt_scd_id_column_name') or "dbt_scd_id" + dbt_updated_at = config.get('dbt_updated_at_column_name') or "dbt_updated_at" + dbt_valid_from = config.get('dbt_valid_from_column_name') or "dbt_valid_from" + dbt_valid_to = config.get('dbt_valid_to_column_name') or "dbt_valid_to" + with snapshot_query as ( @@ -48,7 +53,7 @@ {{ strategy.unique_key }} as dbt_unique_key from {{ target_relation }} - where dbt_valid_to is null + where {{ dbt_valid_to }} is null ), @@ -57,10 +62,10 @@ select *, {{ strategy.unique_key }} as dbt_unique_key, - {{ strategy.updated_at }} as dbt_updated_at, - {{ strategy.updated_at }} as dbt_valid_from, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to, - {{ strategy.scd_id }} as dbt_scd_id + {{ strategy.updated_at }} as {{ dbt_updated_at }}, + {{ strategy.updated_at }} as {{ dbt_valid_from }}, + nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ dbt_valid_to }}, + {{ strategy.scd_id }} as {{ dbt_scd_id }} from snapshot_query ), @@ -70,9 +75,9 @@ select *, {{ strategy.unique_key }} as dbt_unique_key, - {{ strategy.updated_at }} as dbt_updated_at, - {{ strategy.updated_at }} as dbt_valid_from, - {{ strategy.updated_at }} as dbt_valid_to + {{ strategy.updated_at }} as {{ dbt_updated_at }}, + {{ strategy.updated_at }} as {{ dbt_valid_from }}, + {{ strategy.updated_at }} as {{ dbt_valid_to }} from snapshot_query ), @@ -111,7 +116,7 @@ select 'update' as dbt_change_type, source_data.*, - snapshotted_data.dbt_scd_id + snapshotted_data.{{ dbt_scd_id }} from updates_source_data as source_data join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key @@ -128,10 +133,10 @@ select 'delete' as dbt_change_type, source_data.*, - {{ snapshot_get_time() }} as dbt_valid_from, - {{ snapshot_get_time() }} as dbt_updated_at, - {{ snapshot_get_time() }} as dbt_valid_to, - snapshotted_data.dbt_scd_id + {{ snapshot_get_time() }} as {{ dbt_valid_from }}, + {{ snapshot_get_time() }} as {{ dbt_updated_at }}, + {{ snapshot_get_time() }} as {{ dbt_valid_to }}, + snapshotted_data.{{ dbt_scd_id }} from snapshotted_data left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key @@ -155,12 +160,16 @@ {% endmacro %} {% macro default__build_snapshot_table(strategy, sql) %} + dbt_scd_id = config.get('dbt_scd_id_column_name') or "dbt_scd_id" + dbt_updated_at = config.get('dbt_updated_at_column_name') or "dbt_updated_at" + dbt_valid_from = config.get('dbt_valid_from_column_name') or "dbt_valid_from" + dbt_valid_to = config.get('dbt_valid_to_column_name') or "dbt_valid_to" select *, - {{ strategy.scd_id }} as dbt_scd_id, - {{ strategy.updated_at }} as dbt_updated_at, - {{ strategy.updated_at }} as dbt_valid_from, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to + {{ strategy.scd_id }} as {{ dbt_scd_id }}, + {{ strategy.updated_at }} as {{ dbt_updated_at }}, + {{ strategy.updated_at }} as {{ dbt_valid_from }}, + nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ dbt_valid_to }} from ( {{ sql }} ) sbq diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 56798811..a00e23cd 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -7,15 +7,18 @@ {% macro default__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} + dbt_scd_id = config.get("dbt_scd_id_column_name") or "dbt_scd_id" + dbt_valid_to = config.get("dbt_valid_to_column_name") or "dbt_valid_to" + merge into {{ target.render() }} as DBT_INTERNAL_DEST using {{ source }} as DBT_INTERNAL_SOURCE - on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id + on DBT_INTERNAL_SOURCE.{{ dbt_scd_id }} = DBT_INTERNAL_DEST.{{ dbt_scd_id }} when matched - and DBT_INTERNAL_DEST.dbt_valid_to is null + and DBT_INTERNAL_DEST.{{ dbt_valid_to }} is null and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') then update - set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to + set {{ dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ dbt_valid_to }} when not matched and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert' From b765c4979f9fa8471c2e0f911abb70b7230b6123 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 27 Aug 2024 14:04:21 -0400 Subject: [PATCH 02/16] Jinja syntax --- .../materializations/snapshots/helpers.sql | 16 ++++++++-------- .../snapshots/snapshot_merge.sql | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index c218b8d9..937f9945 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -35,10 +35,10 @@ {% endmacro %} {% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} - dbt_scd_id = config.get('dbt_scd_id_column_name') or "dbt_scd_id" - dbt_updated_at = config.get('dbt_updated_at_column_name') or "dbt_updated_at" - dbt_valid_from = config.get('dbt_valid_from_column_name') or "dbt_valid_from" - dbt_valid_to = config.get('dbt_valid_to_column_name') or "dbt_valid_to" + {% set dbt_scd_id = config.get('dbt_scd_id_column_name') or "dbt_scd_id" %} + {% set dbt_updated_at = config.get('dbt_updated_at_column_name') or "dbt_updated_at" %} + {% set dbt_valid_from = config.get('dbt_valid_from_column_name') or "dbt_valid_from" %} + {% set dbt_valid_to = config.get('dbt_valid_to_column_name') or "dbt_valid_to" %} with snapshot_query as ( @@ -160,10 +160,10 @@ {% endmacro %} {% macro default__build_snapshot_table(strategy, sql) %} - dbt_scd_id = config.get('dbt_scd_id_column_name') or "dbt_scd_id" - dbt_updated_at = config.get('dbt_updated_at_column_name') or "dbt_updated_at" - dbt_valid_from = config.get('dbt_valid_from_column_name') or "dbt_valid_from" - dbt_valid_to = config.get('dbt_valid_to_column_name') or "dbt_valid_to" + {% set dbt_scd_id = config.get('dbt_scd_id_column_name') or "dbt_scd_id" %} + {% set dbt_updated_at = config.get('dbt_updated_at_column_name') or "dbt_updated_at" %} + {% set dbt_valid_from = config.get('dbt_valid_from_column_name') or "dbt_valid_from" %} + {% set dbt_valid_to = config.get('dbt_valid_to_column_name') or "dbt_valid_to" %} select *, {{ strategy.scd_id }} as {{ dbt_scd_id }}, diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index a00e23cd..234997d8 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -7,8 +7,8 @@ {% macro default__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} - dbt_scd_id = config.get("dbt_scd_id_column_name") or "dbt_scd_id" - dbt_valid_to = config.get("dbt_valid_to_column_name") or "dbt_valid_to" + {% set dbt_scd_id = config.get("dbt_scd_id_column_name") or "dbt_scd_id" %} + {% set dbt_valid_to = config.get("dbt_valid_to_column_name") or "dbt_valid_to" %} merge into {{ target.render() }} as DBT_INTERNAL_DEST using {{ source }} as DBT_INTERNAL_SOURCE From e4147985f7e1f74b8ce7c8c33e8a13afeb146d2b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 27 Aug 2024 21:57:29 -0400 Subject: [PATCH 03/16] Update strategies.sql, skip column validation for now --- dbt/adapters/base/impl.py | 3 +++ .../macros/materializations/snapshots/snapshot_merge.sql | 4 ++-- .../macros/materializations/snapshots/strategies.sql | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index e0627c47..475738f8 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -748,6 +748,9 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None: :raises InvalidMacroArgType: If the columns are incorrect. """ + # TODO: Need to figure out to validate snapshot target here, probably + # by passing in names of columns. + return if not isinstance(relation, self.Relation): raise MacroArgTypeError( method_name="valid_snapshot_target", diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 234997d8..06fa7235 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -7,8 +7,8 @@ {% macro default__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} - {% set dbt_scd_id = config.get("dbt_scd_id_column_name") or "dbt_scd_id" %} - {% set dbt_valid_to = config.get("dbt_valid_to_column_name") or "dbt_valid_to" %} + {%- set dbt_scd_id = config.get("dbt_scd_id_column_name") or "dbt_scd_id" -%} + {%- set dbt_valid_to = config.get("dbt_valid_to_column_name") or "dbt_valid_to" -%} merge into {{ target.render() }} as DBT_INTERNAL_DEST using {{ source }} as DBT_INTERNAL_SOURCE diff --git a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index d22cc336..4a6fd155 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -52,6 +52,7 @@ {% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} {% set primary_key = config['unique_key'] %} {% set updated_at = config['updated_at'] %} + {% set dbt_valid_from = config.get("dbt_valid_from_column_name") or "dbt_valid_from" %} {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %} {#/* @@ -64,7 +65,7 @@ See https://github.com/dbt-labs/dbt-core/issues/2350 */ #} {% set row_changed_expr -%} - ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }}) + ({{ snapshotted_rel }}.{{ dbt_valid_from }} < {{ current_rel }}.{{ updated_at }}) {%- endset %} {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %} From 5366368b4fa4f25decdc798ee34db566f0ad1a40 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 29 Aug 2024 13:50:53 -0400 Subject: [PATCH 04/16] get snapshot_table_column_names --- .../macros/materializations/snapshots/snapshot.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql index b0fe9222..4b1adc4f 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql @@ -1,5 +1,4 @@ {% materialization snapshot, default %} - {%- set config = model['config'] -%} {%- set target_table = model.get('alias', model.get('name')) -%} @@ -33,6 +32,8 @@ {% else %} + {% set snapshot_table_column_names = config.get("snapshot_table_column_names") %} + {{ adapter.valid_snapshot_target(target_relation) }} {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %} From 23f1c7092e7cc9dcfcc4e3003ea06ba8663d87bb Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 29 Aug 2024 14:55:05 -0400 Subject: [PATCH 05/16] Use 'config' from context instead of model['config'] --- .../materializations/snapshots/snapshot.sql | 2 +- .../materializations/snapshots/strategies.sql | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql index 4b1adc4f..5a4aed2f 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql @@ -23,7 +23,7 @@ {{ run_hooks(pre_hooks, inside_transaction=True) }} {% set strategy_macro = strategy_dispatch(strategy_name) %} - {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", config, target_relation_exists) %} + {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", model['config'], target_relation_exists) %} {% if not target_relation_exists %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index 4a6fd155..386f4033 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -49,11 +49,12 @@ {# Core strategy definitions #} -{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} - {% set primary_key = config['unique_key'] %} - {% set updated_at = config['updated_at'] %} + +{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %} + {% set primary_key = config.get('unique_key') %} + {% set updated_at = config.get('updated_at') %} + {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %} {% set dbt_valid_from = config.get("dbt_valid_from_column_name") or "dbt_valid_from" %} - {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %} {#/* The snapshot relation might not have an {{ updated_at }} value if the @@ -134,11 +135,11 @@ {%- endmacro %} -{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} - {% set check_cols_config = config['check_cols'] %} - {% set primary_key = config['unique_key'] %} - {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %} - {% set updated_at = config.get('updated_at', snapshot_get_time()) %} +{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %} + {% set check_cols_config = config.get('check_cols') %} + {% set primary_key = config.get('unique_key') %} + {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %} + {% set updated_at = config.get('updated_at') or snapshot_get_time() %} {% set column_added = false %} From cdfb049f14c8641bbaf4882e78e72a29f01aa2e5 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 29 Aug 2024 15:52:11 -0400 Subject: [PATCH 06/16] Implement check of custom column names --- dbt/adapters/base/impl.py | 22 ++++++------------- .../materializations/snapshots/snapshot.sql | 3 ++- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index 475738f8..bbb976c0 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -82,7 +82,6 @@ QuoteConfigTypeError, RelationReturnedMultipleResultsError, RenameToNoneAttemptedError, - SnapshotTargetIncompleteError, SnapshotTargetNotSnapshotTableError, UnexpectedNonTimestampError, ) @@ -740,7 +739,9 @@ def get_missing_columns( return [col for (col_name, col) in from_columns.items() if col_name in missing_columns] @available.parse_none - def valid_snapshot_target(self, relation: BaseRelation) -> None: + def valid_snapshot_target( + self, relation: BaseRelation, column_names: Optional[Dict[str, str]] = None + ) -> None: """Ensure that the target relation is valid, by making sure it has the expected columns. @@ -748,9 +749,6 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None: :raises InvalidMacroArgType: If the columns are incorrect. """ - # TODO: Need to figure out to validate snapshot target here, probably - # by passing in names of columns. - return if not isinstance(relation, self.Relation): raise MacroArgTypeError( method_name="valid_snapshot_target", @@ -761,21 +759,15 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None: columns = self.get_columns_in_relation(relation) names = set(c.name.lower() for c in columns) - expanded_keys = ("scd_id", "valid_from", "valid_to") - extra = [] + columns = ("dbt_scd_id", "dbt_valid_from", "dbt_valid_to") missing = [] - for legacy in expanded_keys: - desired = "dbt_" + legacy + for column in columns: + desired = column_names[column] if column_names else column if desired not in names: missing.append(desired) - if legacy in names: - extra.append(legacy) if missing: - if extra: - raise SnapshotTargetIncompleteError(extra, missing) - else: - raise SnapshotTargetNotSnapshotTableError(missing) + raise SnapshotTargetNotSnapshotTableError(missing) @available.parse_none def expand_target_column_types( diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql index 5a4aed2f..362cd153 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql @@ -33,8 +33,9 @@ {% else %} {% set snapshot_table_column_names = config.get("snapshot_table_column_names") %} + {{ log("stcn: " ~ snapshot_table_column_names, info=true) }} - {{ adapter.valid_snapshot_target(target_relation) }} + {{ adapter.valid_snapshot_target(target_relation, snapshot_table_column_names) }} {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %} From 745ae49f236eff057533c823dab5d68c3483d0ae Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 29 Aug 2024 18:33:24 -0400 Subject: [PATCH 07/16] Tweak error message --- dbt/adapters/exceptions/compilation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/exceptions/compilation.py b/dbt/adapters/exceptions/compilation.py index 46ca5219..eb643e6b 100644 --- a/dbt/adapters/exceptions/compilation.py +++ b/dbt/adapters/exceptions/compilation.py @@ -150,7 +150,7 @@ def __init__(self, missing: List): super().__init__(msg=self.get_message()) def get_message(self) -> str: - msg = 'Snapshot target is not a snapshot table (missing "{}")'.format( + msg = 'Snapshot target is not a snapshot table or is missing configured columns (missing "{}")'.format( '", "'.join(self.missing) ) return msg From 2c281a9ee9fa29df36d00c7719af8a87be17e992 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 3 Sep 2024 15:56:24 -0400 Subject: [PATCH 08/16] changie --- .changes/unreleased/Features-20240903-155618.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20240903-155618.yaml diff --git a/.changes/unreleased/Features-20240903-155618.yaml b/.changes/unreleased/Features-20240903-155618.yaml new file mode 100644 index 00000000..fdafceb3 --- /dev/null +++ b/.changes/unreleased/Features-20240903-155618.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allow configuring of snapshot column names +time: 2024-09-03T15:56:18.211492-04:00 +custom: + Author: gshank + Issue: "289" From 79ab8fc1ac684d34f634992f8372eda9e3956ef7 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 3 Sep 2024 16:45:09 -0400 Subject: [PATCH 09/16] Tweak message --- dbt/adapters/exceptions/compilation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/exceptions/compilation.py b/dbt/adapters/exceptions/compilation.py index eb643e6b..91820532 100644 --- a/dbt/adapters/exceptions/compilation.py +++ b/dbt/adapters/exceptions/compilation.py @@ -150,8 +150,10 @@ def __init__(self, missing: List): super().__init__(msg=self.get_message()) def get_message(self) -> str: - msg = 'Snapshot target is not a snapshot table or is missing configured columns (missing "{}")'.format( - '", "'.join(self.missing) + missing = '", "'.join(self.missing) + msg = ( + f'Snapshot target is missing configured columns (missing "{missing}")' + "See https://docs.getdbt.com/docs/build/snapshots#snapshot-meta-fields for more information." ) return msg From 97d01d9fb4a2e3cf080d109c1fe76cec32266520 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 12 Sep 2024 17:27:49 -0400 Subject: [PATCH 10/16] Update to use new column names --- .../materializations/snapshots/helpers.sql | 49 +++++++++---------- .../materializations/snapshots/snapshot.sql | 5 +- .../snapshots/snapshot_merge.sql | 9 ++-- .../materializations/snapshots/strategies.sql | 4 +- 4 files changed, 31 insertions(+), 36 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 565ae34e..f8f6aa09 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -34,12 +34,12 @@ {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }} {% endmacro %} -{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} - {% set dbt_scd_id = config.get('dbt_scd_id_column_name') or "dbt_scd_id" %} - {% set dbt_updated_at = config.get('dbt_updated_at_column_name') or "dbt_updated_at" %} - {% set dbt_valid_from = config.get('dbt_valid_from_column_name') or "dbt_valid_from" %} - {% set dbt_valid_to = config.get('dbt_valid_to_column_name') or "dbt_valid_to" %} +{% macro get_snapshot_table_column_names %} + {{ return {'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at'} }} +{% endmacro %} +{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} + {% set stcn = config.get('snapshot_table_column_names') or get_snapshot_table_columns_names() %} with snapshot_query as ( @@ -53,7 +53,7 @@ {{ strategy.unique_key }} as dbt_unique_key from {{ target_relation }} - where {{ dbt_valid_to }} is null + where {{ stcn.dbt_valid_to }} is null ), @@ -62,10 +62,10 @@ select *, {{ strategy.unique_key }} as dbt_unique_key, - {{ strategy.updated_at }} as {{ dbt_updated_at }}, - {{ strategy.updated_at }} as {{ dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ dbt_valid_to }}, - {{ strategy.scd_id }} as {{ dbt_scd_id }} + {{ strategy.updated_at }} as {{ stcn.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ stcn.dbt_valid_from }}, + nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ stcn.dbt_valid_to }}, + {{ strategy.scd_id }} as {{ stcn.dbt_scd_id }} from snapshot_query ), @@ -75,9 +75,9 @@ select *, {{ strategy.unique_key }} as dbt_unique_key, - {{ strategy.updated_at }} as {{ dbt_updated_at }}, - {{ strategy.updated_at }} as {{ dbt_valid_from }}, - {{ strategy.updated_at }} as {{ dbt_valid_to }} + {{ strategy.updated_at }} as {{ stcn.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ stcn.dbt_valid_from }}, + {{ strategy.updated_at }} as {{ stcn.dbt_valid_to }} from snapshot_query ), @@ -116,7 +116,7 @@ select 'update' as dbt_change_type, source_data.*, - snapshotted_data.{{ dbt_scd_id }} + snapshotted_data.{{ stcn.dbt_scd_id }} from updates_source_data as source_data join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key @@ -133,10 +133,10 @@ select 'delete' as dbt_change_type, source_data.*, - {{ snapshot_get_time() }} as {{ dbt_valid_from }}, - {{ snapshot_get_time() }} as {{ dbt_updated_at }}, - {{ snapshot_get_time() }} as {{ dbt_valid_to }}, - snapshotted_data.{{ dbt_scd_id }} + {{ snapshot_get_time() }} as {{ stcn.dbt_valid_from }}, + {{ snapshot_get_time() }} as {{ stcn.dbt_updated_at }}, + {{ snapshot_get_time() }} as {{ stcn.dbt_valid_to }}, + snapshotted_data.{{ stcn.dbt_scd_id }} from snapshotted_data left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key @@ -160,16 +160,13 @@ {% endmacro %} {% macro default__build_snapshot_table(strategy, sql) %} - {% set dbt_scd_id = config.get('dbt_scd_id_column_name') or "dbt_scd_id" %} - {% set dbt_updated_at = config.get('dbt_updated_at_column_name') or "dbt_updated_at" %} - {% set dbt_valid_from = config.get('dbt_valid_from_column_name') or "dbt_valid_from" %} - {% set dbt_valid_to = config.get('dbt_valid_to_column_name') or "dbt_valid_to" %} + {% set stcn = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} select *, - {{ strategy.scd_id }} as {{ dbt_scd_id }}, - {{ strategy.updated_at }} as {{ dbt_updated_at }}, - {{ strategy.updated_at }} as {{ dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ dbt_valid_to }} + {{ strategy.scd_id }} as {{ stcn.dbt_scd_id }}, + {{ strategy.updated_at }} as {{ stcn.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ stcn.dbt_valid_from }}, + nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ stcn.dbt_valid_to }} from ( {{ sql }} ) sbq diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql index b188f4ab..3fa83015 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql @@ -33,10 +33,9 @@ {% else %} - {% set snapshot_table_column_names = config.get("snapshot_table_column_names") %} - {{ log("stcn: " ~ snapshot_table_column_names, info=true) }} + {% set stcn = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %} - {{ adapter.valid_snapshot_target(target_relation, snapshot_table_column_names) }} + {{ adapter.valid_snapshot_target(target_relation, stcn) }} {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %} {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 06fa7235..52ea39d8 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -7,18 +7,17 @@ {% macro default__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} - {%- set dbt_scd_id = config.get("dbt_scd_id_column_name") or "dbt_scd_id" -%} - {%- set dbt_valid_to = config.get("dbt_valid_to_column_name") or "dbt_valid_to" -%} + {%- set stcn = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%} merge into {{ target.render() }} as DBT_INTERNAL_DEST using {{ source }} as DBT_INTERNAL_SOURCE - on DBT_INTERNAL_SOURCE.{{ dbt_scd_id }} = DBT_INTERNAL_DEST.{{ dbt_scd_id }} + on DBT_INTERNAL_SOURCE.{{ stcn.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ stcn.dbt_scd_id }} when matched - and DBT_INTERNAL_DEST.{{ dbt_valid_to }} is null + and DBT_INTERNAL_DEST.{{ stcn.dbt_valid_to }} is null and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') then update - set {{ dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ dbt_valid_to }} + set {{ stcn.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ stcn.dbt_valid_to }} when not matched and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert' diff --git a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index 386f4033..768e598c 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -54,7 +54,7 @@ {% set primary_key = config.get('unique_key') %} {% set updated_at = config.get('updated_at') %} {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %} - {% set dbt_valid_from = config.get("dbt_valid_from_column_name") or "dbt_valid_from" %} + {% set stcn = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %} {#/* The snapshot relation might not have an {{ updated_at }} value if the @@ -66,7 +66,7 @@ See https://github.com/dbt-labs/dbt-core/issues/2350 */ #} {% set row_changed_expr -%} - ({{ snapshotted_rel }}.{{ dbt_valid_from }} < {{ current_rel }}.{{ updated_at }}) + ({{ snapshotted_rel }}.{{ stcn.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }}) {%- endset %} {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %} From f581cbba65f67e393bc44743f20be8bc3e7ffb0e Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 13 Sep 2024 12:34:05 -0400 Subject: [PATCH 11/16] Fix macro call --- .../macros/materializations/snapshots/helpers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index f8f6aa09..97697dc0 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -34,7 +34,7 @@ {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }} {% endmacro %} -{% macro get_snapshot_table_column_names %} +{% macro get_snapshot_table_column_names() %} {{ return {'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at'} }} {% endmacro %} From 3d7d00d7a54940388638e585e0d4e0aca9250c5e Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 13 Sep 2024 14:30:54 -0400 Subject: [PATCH 12/16] return statement --- .../macros/materializations/snapshots/helpers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 97697dc0..79296977 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -35,7 +35,7 @@ {% endmacro %} {% macro get_snapshot_table_column_names() %} - {{ return {'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at'} }} + {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at'}) }} {% endmacro %} {% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} From 975eeac4fc94ae0e585b41e94f66a40b1ae2beb2 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 16 Sep 2024 14:45:27 -0400 Subject: [PATCH 13/16] Tweak error message. --- dbt/adapters/exceptions/compilation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/exceptions/compilation.py b/dbt/adapters/exceptions/compilation.py index 91820532..d82924e3 100644 --- a/dbt/adapters/exceptions/compilation.py +++ b/dbt/adapters/exceptions/compilation.py @@ -152,7 +152,7 @@ def __init__(self, missing: List): def get_message(self) -> str: missing = '", "'.join(self.missing) msg = ( - f'Snapshot target is missing configured columns (missing "{missing}")' + f'Snapshot target is missing configured columns (missing "{missing}"). ' "See https://docs.getdbt.com/docs/build/snapshots#snapshot-meta-fields for more information." ) return msg From ee657364388adafc16df5a45f01ca43ae39b00dd Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 16 Sep 2024 15:31:16 -0400 Subject: [PATCH 14/16] Fix typo in macro name --- .../macros/materializations/snapshots/helpers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 79296977..fb87f3d9 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -39,7 +39,7 @@ {% endmacro %} {% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} - {% set stcn = config.get('snapshot_table_column_names') or get_snapshot_table_columns_names() %} + {% set stcn = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} with snapshot_query as ( From 87f1140adab2de6223b09a3c5c6db29a376a945c Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 18 Sep 2024 13:41:59 -0400 Subject: [PATCH 15/16] Change stcn to "columns", remove "column" in python method --- dbt/adapters/base/impl.py | 5 ++- .../materializations/snapshots/helpers.sql | 38 +++++++++---------- .../materializations/snapshots/snapshot.sql | 4 +- .../snapshots/snapshot_merge.sql | 8 ++-- .../materializations/snapshots/strategies.sql | 4 +- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index a10111b8..4c7efeef 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -783,9 +783,10 @@ def valid_snapshot_target( columns = self.get_columns_in_relation(relation) names = set(c.name.lower() for c in columns) - columns = ("dbt_scd_id", "dbt_valid_from", "dbt_valid_to") missing = [] - for column in columns: + # Note: we're not checking dbt_updated_at here because it's not + # always present. + for column in ("dbt_scd_id", "dbt_valid_from", "dbt_valid_to"): desired = column_names[column] if column_names else column if desired not in names: missing.append(desired) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index fb87f3d9..8d982855 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -39,7 +39,7 @@ {% endmacro %} {% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} - {% set stcn = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} + {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} with snapshot_query as ( @@ -53,7 +53,7 @@ {{ strategy.unique_key }} as dbt_unique_key from {{ target_relation }} - where {{ stcn.dbt_valid_to }} is null + where {{ columns.dbt_valid_to }} is null ), @@ -62,10 +62,10 @@ select *, {{ strategy.unique_key }} as dbt_unique_key, - {{ strategy.updated_at }} as {{ stcn.dbt_updated_at }}, - {{ strategy.updated_at }} as {{ stcn.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ stcn.dbt_valid_to }}, - {{ strategy.scd_id }} as {{ stcn.dbt_scd_id }} + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}, + {{ strategy.scd_id }} as {{ columns.dbt_scd_id }} from snapshot_query ), @@ -75,9 +75,9 @@ select *, {{ strategy.unique_key }} as dbt_unique_key, - {{ strategy.updated_at }} as {{ stcn.dbt_updated_at }}, - {{ strategy.updated_at }} as {{ stcn.dbt_valid_from }}, - {{ strategy.updated_at }} as {{ stcn.dbt_valid_to }} + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_to }} from snapshot_query ), @@ -116,7 +116,7 @@ select 'update' as dbt_change_type, source_data.*, - snapshotted_data.{{ stcn.dbt_scd_id }} + snapshotted_data.{{ columns.dbt_scd_id }} from updates_source_data as source_data join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key @@ -133,10 +133,10 @@ select 'delete' as dbt_change_type, source_data.*, - {{ snapshot_get_time() }} as {{ stcn.dbt_valid_from }}, - {{ snapshot_get_time() }} as {{ stcn.dbt_updated_at }}, - {{ snapshot_get_time() }} as {{ stcn.dbt_valid_to }}, - snapshotted_data.{{ stcn.dbt_scd_id }} + {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }}, + {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }}, + {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }}, + snapshotted_data.{{ columns.dbt_scd_id }} from snapshotted_data left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key @@ -160,13 +160,13 @@ {% endmacro %} {% macro default__build_snapshot_table(strategy, sql) %} - {% set stcn = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} + {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} select *, - {{ strategy.scd_id }} as {{ stcn.dbt_scd_id }}, - {{ strategy.updated_at }} as {{ stcn.dbt_updated_at }}, - {{ strategy.updated_at }} as {{ stcn.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ stcn.dbt_valid_to }} + {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }} from ( {{ sql }} ) sbq diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql index 3fa83015..fa68a886 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql @@ -33,9 +33,9 @@ {% else %} - {% set stcn = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %} + {% set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %} - {{ adapter.valid_snapshot_target(target_relation, stcn) }} + {{ adapter.valid_snapshot_target(target_relation, columns) }} {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %} {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 52ea39d8..74494ed2 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -7,17 +7,17 @@ {% macro default__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} - {%- set stcn = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%} + {%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%} merge into {{ target.render() }} as DBT_INTERNAL_DEST using {{ source }} as DBT_INTERNAL_SOURCE - on DBT_INTERNAL_SOURCE.{{ stcn.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ stcn.dbt_scd_id }} + on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} when matched - and DBT_INTERNAL_DEST.{{ stcn.dbt_valid_to }} is null + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') then update - set {{ stcn.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ stcn.dbt_valid_to }} + set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} when not matched and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert' diff --git a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index 768e598c..a2bd947f 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -54,7 +54,7 @@ {% set primary_key = config.get('unique_key') %} {% set updated_at = config.get('updated_at') %} {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %} - {% set stcn = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %} + {% set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %} {#/* The snapshot relation might not have an {{ updated_at }} value if the @@ -66,7 +66,7 @@ See https://github.com/dbt-labs/dbt-core/issues/2350 */ #} {% set row_changed_expr -%} - ({{ snapshotted_rel }}.{{ stcn.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }}) + ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }}) {%- endset %} {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %} From cba03b0fecf64e0c63bb5fb6671bc0f639dfcf19 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 18 Sep 2024 17:25:13 -0400 Subject: [PATCH 16/16] Add comments about old model['config'] --- .../macros/materializations/snapshots/snapshot.sql | 2 ++ .../macros/materializations/snapshots/strategies.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql index fa68a886..5daead4c 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql @@ -23,6 +23,8 @@ {{ run_hooks(pre_hooks, inside_transaction=True) }} {% set strategy_macro = strategy_dispatch(strategy_name) %} + {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #} + {# It was a dictionary of config, instead of the config object from the context #} {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", model['config'], target_relation_exists) %} {% if not target_relation_exists %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index a2bd947f..8c086182 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -51,6 +51,7 @@ #} {% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %} + {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #} {% set primary_key = config.get('unique_key') %} {% set updated_at = config.get('updated_at') %} {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %} @@ -136,6 +137,7 @@ {% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %} + {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #} {% set check_cols_config = config.get('check_cols') %} {% set primary_key = config.get('unique_key') %} {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}