Merge pull request #67 from leoebfolsom/lf/fix-incorrect-null-to-miss…

…ing-match bug fix: account for incorrect comparisons involving null values
dbt-labs · Mar 28, 2023 · 3d02772 · 3d02772
2 parents 470b72a + 1dbaaf9
commit 3d02772
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 9 deletions.
diff --git a/integration_tests/seeds/data_compare_all_columns__albertsons_produce.csv b/integration_tests/seeds/data_compare_all_columns__albertsons_produce.csv
@@ -6,4 +6,5 @@ id,fruit,ripeness
 5,orange,orange
 6,,brown
 7,orange,orange
-9,apple,mushy
+9,apple,mushy
+10,apple,
diff --git a/integration_tests/seeds/expected_results__compare_all_columns_where_clause.csv b/integration_tests/seeds/expected_results__compare_all_columns_where_clause.csv
@@ -1,10 +1,13 @@
 primary_key,column_name,perfect_match,null_in_a,null_in_b,missing_from_a,missing_from_b,conflicting_values
 8,ID,false,false,false,false,true,false
 9,ID,false,false,false,true,false,false
+10,ID,false,false,false,true,false,false
 6,FRUIT,false,false,true,false,false,false
 8,FRUIT,false,false,false,false,true,false
 9,FRUIT,false,false,false,true,false,false
+10,FRUIT,false,false,false,true,false,false
 2,RIPENESS,false,false,false,false,false,true
 7,RIPENESS,false,true,false,false,false,false
 8,RIPENESS,false,false,false,false,true,false
-9,RIPENESS,false,false,false,true,false,false
+9,RIPENESS,false,false,false,true,false,false
+10,RIPENESS,false,false,false,true,false,false
diff --git a/macros/compare_column_values_verbose.sql b/macros/compare_column_values_verbose.sql
@@ -20,17 +20,27 @@ b_query as (
             '{{ column_to_compare }}' as column_name,
         {% endif %}
 
-        coalesce(a_query.{{ column_to_compare }} = b_query.{{ column_to_compare }},
-          (a_query.{{ column_to_compare }} is null and b_query.{{ column_to_compare }} is null),
-          false) as perfect_match,
+        coalesce(
+            a_query.{{ column_to_compare }} = b_query.{{ column_to_compare }} and 
+                a_query.{{ primary_key }} is not null and b_query.{{ primary_key }} is not null,
+            (a_query.{{ column_to_compare }} is null and b_query.{{ column_to_compare }} is null),
+            false
+        ) as perfect_match,
         a_query.{{ column_to_compare }} is null and a_query.{{ primary_key }} is not null as null_in_a,
         b_query.{{ column_to_compare }} is null and b_query.{{ primary_key }} is not null as null_in_b,
         a_query.{{ primary_key }} is null as missing_from_a,
         b_query.{{ primary_key }} is null as missing_from_b,
-        coalesce(a_query.{{ column_to_compare }} != b_query.{{ column_to_compare }} and
-            (a_query.{{ column_to_compare }} is not null or b_query.{{ column_to_compare }} is not null), false)
-          as conflicting_values
-           -- considered a conflict if the values do not match AND at least one of the values is not null.
+        coalesce(
+            a_query.{{ primary_key }} is not null and b_query.{{ primary_key }} is not null and 
+            -- ensure that neither value is missing before considering it a conflict
+            (
+                a_query.{{ column_to_compare }} != b_query.{{ column_to_compare }} or -- two not-null values that do not match
+                (a_query.{{ column_to_compare }} is not null and b_query.{{ column_to_compare }} is null) or -- null in b and not null in a
+                (a_query.{{ column_to_compare }} is null and b_query.{{ column_to_compare }} is not null) -- null in a and not null in b
+            ), 
+            false
+        ) as conflicting_values
+        -- considered a conflict if the values do not match AND at least one of the values is not null.
 
     from a_query