Skip to content

Commit

Permalink
Merge pull request #81 from Snowflake-Labs/80-additional-model-exists…
Browse files Browse the repository at this point in the history 
…-verification

80 additional model exists verification
  • Loading branch information
@sfc-gh-dflippo
sfc-gh-dflippo authored Sep 30, 2024
2 parents 6b20f14 + 3fb434e commit b8a1a08
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 53 deletions.
2 changes: 1 addition & 1 deletion dbt_project.yml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

name: 'dbt_constraints'
version: '1.0.3'
version: '1.0.4'
config-version: 2

# These macros depend on the results and graph objects in dbt >=0.19.0
Expand Down
2 changes: 2 additions & 0 deletions integration_tests/dbt_project.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ vars:
dbt_constraints_sources_uk_enabled: true
dbt_constraints_sources_fk_enabled: true

on-run-start:
- "drop table if exists dim_orders"

models:
+materialized: table
Expand Down
2 changes: 1 addition & 1 deletion integration_tests/packages.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@ packages:

# We will test unique keys based on dbt_utils.unique_combination_of_columns
- package: dbt-labs/dbt_utils
version: [">=1.2.0"]
version: [">=1.3.0"]

- local: ../
130 changes: 79 additions & 51 deletions macros/create_constraints.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,16 +174,15 @@
{#- This macro checks if a test or its model is selected -#}
{%- macro test_selected(test_model) -%}

{%- if test_model.unique_id and test_model.unique_id in selected_resources -%}
{%- if test_model.unique_id in selected_resources -%}
{{ return("TEST_SELECTED") }}
{%- endif -%}
{%- if test_model.attached_node and test_model.attached_node in selected_resources -%} -%}
{%- if test_model.attached_node in selected_resources -%} -%}
{{ return("MODEL_SELECTED") }}
{%- endif -%}

{#- Check if a PK/UK should be created because it is referenced by a selected FK -#}
{%- if test_model.test_metadata
and test_model.test_metadata.name in ("primary_key", "unique_key", "unique_combination_of_columns", "unique") -%}
{%- if test_model.test_metadata.name in ("primary_key", "unique_key", "unique_combination_of_columns", "unique") -%}
{%- set pk_test_args = test_model.test_metadata.kwargs -%}
{%- set pk_test_columns = [] -%}
{%- if pk_test_args.column_names -%}
Expand Down Expand Up @@ -230,7 +229,10 @@
{%- for res in results
if res.node.config.materialized == "test"
and res.node.unique_id == test_model.unique_id -%}
{%- if res.failures > 0 -%}
{%- if res.failures == None -%}
{#- Set '' if we do not know if there is a test failure -#}
{{ return('') }}
{%- elif res.failures > 0 -%}
{#- Set NORELY if there is a test failure -#}
{{ return('NORELY') }}
{%- elif res.failures == 0 -%}
Expand Down Expand Up @@ -261,10 +263,17 @@
{#- This macro is called internally and passed which constraint types to create. -#}
{%- macro create_constraints_by_type(constraint_types, quote_columns, lookup_cache) -%}

{#- Loop through the metadata and find all tests that match the constraint_types -#}
{#- Loop through the metadata and find all tests that match the constraint_types and have all the fields we check for tests -#}
{%- for test_model in graph.nodes.values() | selectattr("resource_type", "equalto", "test")
if test_model.test_metadata
and test_model.test_metadata.kwargs
and test_model.test_metadata.name
and test_model.test_metadata.name is in( constraint_types )
and test_model.unique_id
and test_model.attached_node
and test_model.depends_on
and test_model.depends_on.nodes
and test_model.config
and test_model.config.enabled
and test_model.config.get("dbt_constraints_enabled", true) -%}

Expand Down Expand Up @@ -300,7 +309,8 @@
{#- Find the table models that are referenced by this test. -#}
{%- for table_node in test_model.depends_on.nodes -%}
{%- for node in graph.nodes.values() | selectattr("unique_id", "equalto", table_node)
if node.config.get("materialized", "other") not in ("view", "ephemeral", "dynamic_table")
if node.config
and node.config.get("materialized", "other") not in ("view", "ephemeral", "dynamic_table")
and ( node.resource_type in ("model", "snapshot", "seed")
or ( node.resource_type == "source" and var('dbt_constraints_sources_enabled', false)
and ( ( var('dbt_constraints_sources_pk_enabled', false) and test_name in("primary_key") )
Expand Down Expand Up @@ -340,21 +350,25 @@
) }}
{%- endif -%}

{%- set table_relation = api.Relation.create(
{%- set table_relation = adapter.get_relation(
database=table_models[0].database,
schema=table_models[0].schema,
identifier=table_models[0].alias ) -%}
{%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%}
{%- if test_name == "primary_key" -%}
{%- if dbt_constraints.adapter_supports_rely_norely("not_null") == true -%}
{%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%}
{%- if table_relation and table_relation.is_table -%}
{%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%}
{%- if test_name == "primary_key" -%}
{%- if dbt_constraints.adapter_supports_rely_norely("not_null") == true -%}
{%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%}
{%- endif -%}
{%- do dbt_constraints.create_primary_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- else -%}
{%- do dbt_constraints.create_unique_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- endif -%}
{%- do dbt_constraints.create_primary_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- else -%}
{%- do dbt_constraints.create_unique_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- do log("Skipping primary/unique key because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%}
{%- endif -%}
{%- else -%}
{%- do log("Skipping primary/unique key because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%}
{%- do log("Skipping primary/unique key because the table was not found in the database: " ~ table_models[0].name, info=true) -%}
{%- endif -%}

{#- We only create FK if there are two models referenced by the test
Expand All @@ -367,50 +381,60 @@

{%- if fk_model and pk_model -%}

{%- set fk_table_relation = api.Relation.create(
{%- set fk_table_relation = adapter.get_relation(
database=fk_model.database,
schema=fk_model.schema,
identifier=fk_model.alias) -%}

{%- set pk_table_relation = api.Relation.create(
{%- set pk_table_relation = adapter.get_relation(
database=pk_model.database,
schema=pk_model.schema,
identifier=pk_model.alias) -%}

{# Attempt to identify parameters we can use for the column names #}
{%- set pk_column_names = [] -%}
{%- if test_parameters.pk_column_names -%}
{%- set pk_column_names = test_parameters.pk_column_names -%}
{%- elif test_parameters.field -%}
{%- set pk_column_names = [test_parameters.field] -%}
{%- elif test_parameters.pk_column_name -%}
{%- set pk_column_names = [test_parameters.pk_column_name] -%}
{%- else -%}
{{ exceptions.raise_compiler_error(
"`pk_column_names`, `pk_column_name`, or `field` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters
) }}
{%- endif -%}
{%- if fk_table_relation and pk_table_relation and fk_table_relation.is_table and pk_table_relation.is_table-%}
{# Attempt to identify parameters we can use for the column names #}
{%- set pk_column_names = [] -%}
{%- if test_parameters.pk_column_names -%}
{%- set pk_column_names = test_parameters.pk_column_names -%}
{%- elif test_parameters.field -%}
{%- set pk_column_names = [test_parameters.field] -%}
{%- elif test_parameters.pk_column_name -%}
{%- set pk_column_names = [test_parameters.pk_column_name] -%}
{%- else -%}
{{ exceptions.raise_compiler_error(
"`pk_column_names`, `pk_column_name`, or `field` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters
) }}
{%- endif -%}

{%- set fk_column_names = [] -%}
{%- if test_parameters.fk_column_names -%}
{%- set fk_column_names = test_parameters.fk_column_names -%}
{%- elif test_parameters.column_name -%}
{%- set fk_column_names = [test_parameters.column_name] -%}
{%- elif test_parameters.fk_column_name -%}
{%- set fk_column_names = [test_parameters.fk_column_name] -%}
{%- else -%}
{{ exceptions.raise_compiler_error(
"`fk_column_names`, `fk_column_name`, or `column_name` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters
) }}
{%- endif -%}
{%- set fk_column_names = [] -%}
{%- if test_parameters.fk_column_names -%}
{%- set fk_column_names = test_parameters.fk_column_names -%}
{%- elif test_parameters.column_name -%}
{%- set fk_column_names = [test_parameters.column_name] -%}
{%- elif test_parameters.fk_column_name -%}
{%- set fk_column_names = [test_parameters.fk_column_name] -%}
{%- else -%}
{{ exceptions.raise_compiler_error(
"`fk_column_names`, `fk_column_name`, or `column_name` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters
) }}
{%- endif -%}

{%- if not dbt_constraints.table_columns_all_exist(pk_table_relation, pk_column_names, lookup_cache) -%}
{%- do log("Skipping foreign key because a physical column was not found on the pk table: " ~ pk_model.name ~ " " ~ pk_column_names, info=true) -%}
{%- elif not dbt_constraints.table_columns_all_exist(fk_table_relation, fk_column_names, lookup_cache) -%}
{%- do log("Skipping foreign key because a physical column was not found on the fk table: " ~ fk_model.name ~ " " ~ fk_column_names, info=true) -%}
{%- if not dbt_constraints.table_columns_all_exist(pk_table_relation, pk_column_names, lookup_cache) -%}
{%- do log("Skipping foreign key because a physical column was not found on the pk table: " ~ pk_model.name ~ " " ~ pk_column_names, info=true) -%}
{%- elif not dbt_constraints.table_columns_all_exist(fk_table_relation, fk_column_names, lookup_cache) -%}
{%- do log("Skipping foreign key because a physical column was not found on the fk table: " ~ fk_model.name ~ " " ~ fk_column_names, info=true) -%}
{%- else -%}
{%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- endif -%}
{%- else -%}
{%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- if fk_model == None or not fk_table_relation.is_table -%}
{%- do log("Skipping foreign key to " ~ pk_model.alias ~ " because the child table was not found in the database: " ~ fk_model.alias, info=true) -%}
{%- endif -%}
{%- if pk_model == None or not pk_model.is_table -%}
{%- do log("Skipping foreign key on " ~ fk_model.alias ~ " because the parent table was not found in the database: " ~ pk_model.alias, info=true) -%}
{%- endif -%}
{%- endif -%}

{%- else -%}
{%- do log("Skipping foreign key because a we couldn't find the child table: model=" ~ test_model.attached_node ~ " or source", info=true) -%}
{%- endif -%}
Expand All @@ -434,15 +458,19 @@
) }}
{%- endif -%}

{%- set table_relation = api.Relation.create(
{%- set table_relation = adapter.get_relation(
database=table_models[0].database,
schema=table_models[0].schema,
identifier=table_models[0].alias ) -%}

{%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%}
{%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%}
{%- if table_relation and table_relation.is_table -%}
{%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%}
{%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%}
{%- else -%}
{%- do log("Skipping not null constraint because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%}
{%- endif -%}
{%- else -%}
{%- do log("Skipping not null constraint because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%}
{%- do log("Skipping not null constraint because the table was not found in the database: " ~ table_models[0].name, info=true) -%}
{%- endif -%}

{%- endif -%}
Expand Down

0 comments on commit b8a1a08

Please sign in to comment.