From b5b2df72a37877957c603fc97c16d654d9865530 Mon Sep 17 00:00:00 2001 From: chenp-eleos <152792875+chenp-eleos@users.noreply.github.com> Date: Thu, 21 Nov 2024 21:10:30 +0200 Subject: [PATCH 1/2] Update incremental.sql Generate Spark-compatible tmp_table_suffix for incremental materializations with iceberg --- .../materializations/models/incremental/incremental.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql b/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql index c18ac681..48c1e169 100644 --- a/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql +++ b/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql @@ -21,9 +21,12 @@ {% endif %} {% if unique_tmp_table_suffix == True and table_type == 'iceberg' %} - {% set tmp_table_suffix = adapter.generate_unique_temporary_table_suffix() %} + {% set raw_suffix = adapter.generate_unique_temporary_table_suffix() %} + {% set tmp_table_suffix = raw_suffix.replace('-', '_') %} + {% else %} + {% set tmp_table_suffix = '__dbt_tmp' %} {% endif %} - + {% set old_tmp_relation = adapter.get_relation(identifier=target_relation.identifier ~ tmp_table_suffix, schema=schema, database=database) %} From dd2e3e66e052b44cfaf94d67f57a78be252174c3 Mon Sep 17 00:00:00 2001 From: chenp-eleos <152792875+chenp-eleos@users.noreply.github.com> Date: Thu, 21 Nov 2024 21:16:01 +0200 Subject: [PATCH 2/2] Update incremental.sql Generate a unique tmp table suffix if required for incremental materializations. Generate Spark-compatible tmp_table_suffix for incremental materializations. --- .../models/incremental/incremental.sql | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql b/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql index 48c1e169..a6a13e29 100644 --- a/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql +++ b/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql @@ -13,14 +13,8 @@ {% set temp_schema = config.get('temp_schema') %} {% set target_relation = this.incorporate(type='table') %} {% set existing_relation = load_relation(this) %} - -- If using insert_overwrite on Hive table, allow to set a unique tmp table suffix - {% if unique_tmp_table_suffix == True and strategy == 'insert_overwrite' and table_type == 'hive' %} - {% set tmp_table_suffix = adapter.generate_unique_temporary_table_suffix() %} - {% else %} - {% set tmp_table_suffix = '__dbt_tmp' %} - {% endif %} - - {% if unique_tmp_table_suffix == True and table_type == 'iceberg' %} + -- Generate a unique tmp table suffix if required + {% if unique_tmp_table_suffix == True %} {% set raw_suffix = adapter.generate_unique_temporary_table_suffix() %} {% set tmp_table_suffix = raw_suffix.replace('-', '_') %} {% else %}