allow deletes

Brigad · Feb 6, 2024 · 9018193 · 9018193
1 parent fc075f6
commit 9018193
Showing 1 changed file with 43 additions and 52 deletions.
diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/incremental.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/incremental.sql
@@ -62,73 +62,64 @@ and ((select max(cast({{ col_emitted_at }} as {{ type_timestamp_with_timezone()
 {% endmacro %}
 
 {% macro databricks__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
-  {%- set drop_unallowed = true -%}
-  {% if remove_columns %}
-    {% if relation.is_delta %}
-    {% set platform_name = 'Delta Lake' %}
-      {%- set tblproperties = config.get('tblproperties') -%}
-      {%- if not tblproperties -%}
-        {%- set tblproperties = fetch_tbl_properties(relation) -%}
-      {%- endif -%}
-      {%- set good_predicates = 0 -%}
-      {%- for prop in tblproperties -%}
-        {%- if prop == 'delta.minReaderVersion' and tblproperties[prop] == 2 %-}
-          {%- set good_predicates = good_predicates + 1 -%}
-        {%- elif prop == 'delta.minWriterVersion' and tblproperties[prop] == 5 -%}
-          {%- set good_predicates = good_predicates + 1 -%}
-        {%- endif -%}
-        {%- elif prop == 'delta.columnMapping.mode' and tblproperties[prop] == 'name' -%}
-          {%- set good_predicates = good_predicates + 1 -%}
-        {%- endif -%}
-      {%- endfor -%}
-      {%- if good_predicates == 3 -%}
-        {%- set drop_unallowed = false -%}
-
-      {%- endif -%}
-    {%- endif -%}
-    {% elif relation.is_iceberg %}
-      {% set platform_name = 'Iceberg' %}
-    {% else %}
-      {% set platform_name = 'Apache Spark' %}
+    {%- set drop_unallowed = true -%}
+    {% if remove_columns %}
+        {% if relation.is_delta %}
+            {% set platform_name = 'Delta Lake' %}
+            {%- set tblproperties = config.get('tblproperties') -%}
+            {%- if not tblproperties -%}
+                {%- set tblproperties = fetch_tbl_properties(relation) -%}
+            {%- endif -%}
+            {%- set good_predicates = namespace(predicates=0) -%}
+            {%- for prop in tblproperties -%}
+                {%- if prop == 'delta.minReaderVersion' and tblproperties[prop] == '2' -%}
+                    {%- set good_predicates.predicates = good_predicates.predicates + 1 -%}
+                    {{ log("Incremented good predicates to " ~ good_predicates, True) }}
+                {%- elif prop == 'delta.minWriterVersion' and tblproperties[prop] == '5' -%}
+                    {%- set good_predicates.predicates = good_predicates.predicates + 1 -%}
+                {%- elif prop == 'delta.columnMapping.mode' and tblproperties[prop] == 'name' -%}
+                    {%- set good_predicates.predicates = good_predicates.predicates + 1 -%}
+                {%- endif -%}
+            {%- endfor -%}
+            {%- if good_predicates.predicates == 3 -%}
+                {%- set drop_unallowed = false -%}
+            {%- endif -%}
+        {% elif relation.is_iceberg %} {% set platform_name = 'Iceberg' %}
+        {% else %} {% set platform_name = 'Apache Spark' %}
+        {% endif %}
+        {% if drop_unallowed %}
+            {% if relation.is_delta %}
+                {{ exceptions.raise_compiler_error('To allow dropping column, ensure you have tblproperties set as followed : delta.minReaderVersion: 2 / delta.minWriterVersion: 5 / delta.columnMapping.mode: name') }}
+            {% endif %}
+            {{ exceptions.raise_compiler_error(platform_name + ' does not support dropping columns from tables') }}
+        {% endif %}
     {% endif %}
-    {% if drop_unallowed %}
-    {{ exceptions.raise_compiler_error(platform_name + ' does not support dropping columns from tables') }}
-      {% if relation.is_delta %}
-        {{ exceptions.raise_compiler_error('To allow dropping column, ensure you have tblproperties set as followed : delta.minReaderVersion: 2 / delta.minWriterVersion: 5 / delta.columnMapping.mode: name') }}
-      {% endif %}
-    {% endif %}
-  {% endif %}
-
-  {% if drop_columns is none %}
-    {% set drop_columns = [] %}
-  {% endif %}
 
-  {% set drop_sql -%}
+    {% if remove_columns is none %} {% set remove_columns = [] %} {% endif %}
+    {% set drop_sql -%}
 
     alter {{ relation.type }} {{ relation }}
 
-    {% if drop_columns %} drop columns {% endif %}
-        {% for column in drop_columns %}
+    drop columns
+        {% for column in remove_columns %}
             {{ column.name }} {{ ',' if not loop.last }}
         {% endfor %}
-  {%- endset -%}
+    {%- endset -%}
 
-  {% if add_columns is none %}
-    {% set add_columns = [] %}
-  {% endif %}
+    {% if add_columns is none %} {% set add_columns = [] %} {% endif %}
 
-  {% set add_sql -%}
+    {% set add_sql -%}
 
      alter {{ relation.type }} {{ relation }}
 
-       {% if add_columns %} add columns {% endif %}
+       add columns
             {% for column in add_columns %}
                {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}
             {% endfor %}
 
-  {%- endset -%}
+    {%- endset -%}
 
-  {% do run_query(drop_sql) %}
-  {% do run_query(add_sql) %}
+    {% if remove_columns %} {% do run_query(drop_sql) %} {% endif %}
+    {% if add_columns %} {% do run_query(add_sql) %} {% endif %}
 
-{% endmacro %}
+{% endmacro %}