Merge pull request #76 from Snowflake-Labs/create-constraints-for-fai…

…led-skipped-tests

Updated to avoid including FK constraints when the parent table is selected but not the child

dbt_project.yml

@@ -1,11 +1,12 @@
 
 name: 'dbt_constraints'
-version: '1.0.1'
+version: '1.0.3'
 config-version: 2
 
 # These macros depend on the results and graph objects in dbt >=0.19.0
 # and has been tested with dbt >=1.0.0
-require-dbt-version: ">=1.0.0"
+# v1.0.3 adds usage of the attached_node attribute added in dbt 1.5.x
+require-dbt-version: ">=1.5.0"
 
 profile: 'dbt_constraints'
 

docs/catalog.json

@@ -1 +1 @@
-{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json", "dbt_version": "1.3.2", "generated_at": "2023-01-20T04:30:03.174118Z", "invocation_id": "134cbac7-4598-4147-95e0-4a1ee1fd6018", "env": {}}, "nodes": {}, "sources": {}, "errors": null}
+{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json", "dbt_version": "1.8.5", "generated_at": "2024-08-19T18:34:12.783856Z", "invocation_id": "6248ccf8-48d0-470c-b010-2db8eb51c482", "env": {}}, "nodes": {}, "sources": {}, "errors": null}

docs/graph_summary.json

@@ -0,0 +1 @@
+{"_invocation_id": "6248ccf8-48d0-470c-b010-2db8eb51c482", "linked": {"0": {"name": "operation.dbt_constraints.dbt_constraints-on-run-end-0", "type": "operation"}}}

docs/run_results.json

@@ -1 +1 @@
-{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v4.json", "dbt_version": "1.3.2", "generated_at": "2023-01-20T04:30:03.161471Z", "invocation_id": "134cbac7-4598-4147-95e0-4a1ee1fd6018", "env": {}}, "results": [{"status": "success", "timing": [{"name": "compile", "started_at": "2023-01-20T04:30:03.148101Z", "completed_at": "2023-01-20T04:30:03.158655Z"}, {"name": "execute", "started_at": "2023-01-20T04:30:03.159021Z", "completed_at": "2023-01-20T04:30:03.159048Z"}], "thread_id": "Thread-1", "execution_time": 0.012279033660888672, "adapter_response": {}, "message": null, "failures": null, "unique_id": "operation.dbt_constraints.dbt_constraints-on-run-end-0"}], "elapsed_time": 0.02072310447692871, "args": {"write_json": true, "use_colors": true, "printer_width": 80, "version_check": true, "partial_parse": true, "static_parser": true, "profiles_dir": "/Users/dflippo/.dbt", "send_anonymous_usage_stats": true, "event_buffer_size": 100000, "quiet": false, "no_print": false, "cache_selected_only": false, "compile": true, "which": "generate", "rpc_method": "docs.generate", "indirect_selection": "eager"}}
+{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v6.json", "dbt_version": "1.8.5", "generated_at": "2024-08-19T18:34:12.756239Z", "invocation_id": "6248ccf8-48d0-470c-b010-2db8eb51c482", "env": {}}, "results": [{"status": "success", "timing": [{"name": "compile", "started_at": "2024-08-19T18:34:12.748983Z", "completed_at": "2024-08-19T18:34:12.754902Z"}, {"name": "execute", "started_at": "2024-08-19T18:34:12.755039Z", "completed_at": "2024-08-19T18:34:12.755046Z"}], "thread_id": "Thread-1 (worker)", "execution_time": 0.006446123123168945, "adapter_response": {}, "message": null, "failures": null, "unique_id": "operation.dbt_constraints.dbt_constraints-on-run-end-0", "compiled": true, "compiled_code": "", "relation_name": null}], "elapsed_time": 0.00990605354309082, "args": {"show_resource_report": false, "log_level": "info", "populate_cache": true, "warn_error_options": {"include": [], "exclude": []}, "version_check": true, "use_colors_file": true, "exclude": [], "quiet": false, "printer_width": 80, "defer": false, "source_freshness_run_project_hooks": false, "write_json": true, "require_explicit_package_overrides_for_builtin_materializations": true, "macro_debugging": false, "log_format_file": "debug", "profiles_dir": "/Users/dflippo/.dbt", "static_parser": true, "invocation_command": "dbt docs generate", "strict_mode": false, "log_path": "/Users/dflippo/Documents/GitHub/Snowflake-Labs/dbt_constraints/logs", "partial_parse": true, "project_dir": "/Users/dflippo/Documents/GitHub/Snowflake-Labs/dbt_constraints", "enable_legacy_logger": false, "send_anonymous_usage_stats": true, "print": true, "empty_catalog": false, "indirect_selection": "eager", "log_level_file": "debug", "vars": {}, "log_file_max_bytes": 10485760, "log_format": "default", "favor_state": false, "introspect": true, "use_colors": true, "partial_parse_file_diff": true, "require_resource_names_without_spaces": false, "compile": true, "cache_selected_only": false, "select": [], "static": false, "which": "generate"}}

docs/semantic_manifest.json

@@ -0,0 +1 @@
+{"semantic_models": [], "metrics": [], "project_configuration": {"time_spine_table_configurations": [], "metadata": null, "dsi_package_version": {"major_version": "0", "minor_version": "5", "patch_version": "1"}}, "saved_queries": []}

integration_tests/dbt_project.yml

@@ -52,8 +52,8 @@ seeds:
   #+full_refresh: false
 
 tests:
-  +dbt_constraints_integration_tests:
-    #+always_create_constraint: true
+  dbt_constraints_integration_tests:
+    +always_create_constraint: true
     # These configuration settings disable running tests or just constraints by path
     # +enabled: false
     #+dbt_constraints_enabled: false

integration_tests/packages.yml

@@ -2,6 +2,6 @@ packages:
 
   # We will test unique keys based on dbt_utils.unique_combination_of_columns
   - package: dbt-labs/dbt_utils
-    version: [">=0.8.0"]
+    version: [">=1.2.0"]
 
   - local: ../

macros/create_constraints.sql

@@ -174,10 +174,10 @@
 {#- This macro checks if a test or its model is selected -#}
 {%- macro test_selected(test_model) -%}
 
-    {%- if test_model.unique_id in selected_resources -%}
+    {%- if test_model.unique_id and test_model.unique_id in selected_resources -%}
         {{ return("TEST_SELECTED") }}
     {%- endif -%}
-    {%- if dbt_constraints.lists_intersect(test_model.depends_on.nodes, selected_resources) -%}
+    {%- if test_model.attached_node and test_model.attached_node in selected_resources -%} -%}
         {{ return("MODEL_SELECTED") }}
     {%- endif -%}
 
@@ -196,9 +196,9 @@
         {%- for fk_model in graph.nodes.values() | selectattr("resource_type", "equalto", "test")
                 if  fk_model.test_metadata
                 and fk_model.test_metadata.name in ("foreign_key", "relationships")
-                and dbt_constraints.lists_intersect(test_model.depends_on.nodes, fk_model.depends_on.nodes)
-                and ( fk_model.unique_id in selected_resources
-                    or dbt_constraints.lists_intersect(fk_model.depends_on.nodes, selected_resources) )  -%}
+                and test_model.attached_node in fk_model.depends_on.nodes
+                and ( (fk_model.unique_id and fk_model.unique_id in selected_resources)
+                    or (fk_model.attached_node and fk_model.attached_node in selected_resources) ) -%}
             {%- set fk_test_args = fk_model.test_metadata.kwargs -%}
             {%- set fk_test_columns = [] -%}
             {%- if fk_test_args.pk_column_names -%}
@@ -362,24 +362,8 @@
             {%- elif 2 == table_models|count
                 and test_name in( "foreign_key", "relationships") -%}
 
-                {%- set fk_model = none -%}
-                {%- set pk_model = none -%}
-                {%- set fk_model_names = modules.re.findall( "(models|snapshots|seeds)\W+(\w+)" , test_model.file_key_name)  -%}
-                {%- set fk_source_names = modules.re.findall( "source\W+(\w+)\W+(\w+)" , test_parameters.model)  -%}
-
-                {%- if 1 == fk_model_names | count -%}
-                    {%- set fk_model = table_models | selectattr("name", "equalto", fk_model_names[0][1]) | first -%}
-                    {%- set pk_model = table_models | rejectattr("name", "equalto", fk_model_names[0][1]) | first -%}
-                {%- elif 1 == fk_source_names | count  -%}
-                    {%- if table_models[0].source_name == fk_source_names[0][0] and table_models[0].name == fk_source_names[0][1] -%}
-                        {%- set fk_model = table_models[0] -%}
-                        {%- set pk_model = table_models[1] -%}
-                    {%- else  -%}
-                        {%- set fk_model = table_models[1] -%}
-                        {%- set pk_model = table_models[0] -%}
-                    {%- endif -%}
-                {%- endif -%}
-                {# {%- set fk_model_name = test_model.file_key_name |replace("models.", "") -%} #}
+                {%- set fk_model = table_models | selectattr("unique_id", "equalto", test_model.attached_node) | first -%}
+                {%- set pk_model = table_models | rejectattr("unique_id", "equalto", test_model.attached_node) | first -%}
 
                 {%- if fk_model and pk_model -%}
 
@@ -428,7 +412,7 @@
                         {%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
                     {%- endif -%}
                 {%- else  -%}
-                    {%- do log("Skipping foreign key because a we couldn't find the child table: model=" ~ fk_model_names ~ " or source=" ~ fk_source_names, info=true) -%}
+                    {%- do log("Skipping foreign key because a we couldn't find the child table: model=" ~ test_model.attached_node ~ " or source", info=true) -%}
                 {%- endif -%}
 
             {#- We only create NN if there is one model referenced by the test
@@ -518,11 +502,3 @@
         {{ return(false) }}
     {%- endif -%}
 {%- endmacro -%}
-
-{# This macro allows us to compare two lists to see if they intersect #}
-{%- macro lists_intersect(listA, listB) -%}
-    {%- for valueFromA in listA if valueFromA in listB  -%}
-        {{ return(true) }}
-    {% endfor %}
-    {{ return(false) }}
-{%- endmacro -%}