From aeea60265b033a863d74c2c1bca8c330a765f6ed Mon Sep 17 00:00:00 2001 From: Jeremynadal33 <36603814+Jeremynadal33@users.noreply.github.com> Date: Mon, 26 Aug 2024 07:13:18 +0200 Subject: [PATCH] Fix table materialization for Delta models (#420) * added delta_table_replace materialization * fixed table materialization for delta format * removed or replace without as clause --- CHANGELOG.md | 3 ++- dbt/adapters/glue/impl.py | 5 ++++- dbt/include/glue/macros/adapters.sql | 10 +++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eddb3010..4869b4d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ -## New versio +## New version - Fix session provisioning timeout and delay handling +- Fix table materialization for Delta models ## v1.8.1 - Fix typo in README.md diff --git a/dbt/adapters/glue/impl.py b/dbt/adapters/glue/impl.py index fac57744..0db6226e 100644 --- a/dbt/adapters/glue/impl.py +++ b/dbt/adapters/glue/impl.py @@ -187,11 +187,14 @@ def get_relation(self, database, schema, identifier): DatabaseName=schema, Name=identifier ) + is_delta = response.get('Table').get("Parameters").get("spark.sql.sources.provider") == "delta" + relations = self.Relation.create( database=schema, schema=schema, identifier=identifier, - type=self.relation_type_map.get(response.get("Table", {}).get("TableType", "Table")) + type=self.relation_type_map.get(response.get("Table", {}).get("TableType", "Table")), + is_delta=is_delta ) logger.debug(f"""schema : {schema} identifier : {identifier} diff --git a/dbt/include/glue/macros/adapters.sql b/dbt/include/glue/macros/adapters.sql index b1613442..b79f251c 100644 --- a/dbt/include/glue/macros/adapters.sql +++ b/dbt/include/glue/macros/adapters.sql @@ -62,10 +62,18 @@ {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%} {%- set table_properties = config.get('table_properties', default={}) -%} + {%- set create_statement_string -%} + {% if file_format in ['delta', 'iceberg'] -%} + create or replace table + {%- else -%} + create table + {% endif %} + {%- endset %} + {% if temporary -%} {{ create_temporary_view(relation, sql) }} {%- else -%} - create table {{ relation }} + {{ create_statement_string }} {{ relation }} {% set contract_config = config.get('contract') %} {% if contract_config.enforced %} {{ get_assert_columns_equivalent(sql) }}