Skip to content

Commit

Permalink
fixed table materialization for delta format
Browse files Browse the repository at this point in the history
  • Loading branch information
Jeremynadal33 committed Aug 8, 2024
1 parent 3e43926 commit a855e39
Show file tree
Hide file tree
Showing 4 changed files with 15 additions and 46 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
## New version
- Fix session provisioning timeout and delay handling
- Add delta_table_replace materialization
- Fix table materialization for Delta models

## v1.8.1
- Fix typo in README.md
Expand Down
7 changes: 5 additions & 2 deletions dbt/adapters/glue/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,11 +187,14 @@ def get_relation(self, database, schema, identifier):
DatabaseName=schema,
Name=identifier
)
is_delta = response.get('Table').get("Parameters").get("spark.sql.sources.provider") == "delta"

relations = self.Relation.create(
database=schema,
schema=schema,
identifier=identifier,
type=self.relation_type_map.get(response.get("Table", {}).get("TableType", "Table"))
type=self.relation_type_map.get(response.get("Table", {}).get("TableType", "Table")),
is_delta=is_delta
)
logger.debug(f"""schema : {schema}
identifier : {identifier}
Expand Down Expand Up @@ -638,7 +641,7 @@ def delta_create_table(self, target_relation, request, primary_key, partition_ke
location = custom_location

create_table_query = f"""
CREATE TABLE {table_name}
CREATE OR REPLACE TABLE {table_name}
USING delta
LOCATION '{location}'
"""
Expand Down
10 changes: 9 additions & 1 deletion dbt/include/glue/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,18 @@
{%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}
{%- set table_properties = config.get('table_properties', default={}) -%}

{%- set create_statement_string -%}
{% if file_format in ['delta', 'iceberg'] -%}
create or replace table
{%- else -%}
create table
{% endif %}
{%- endset %}

{% if temporary -%}
{{ create_temporary_view(relation, sql) }}
{%- else -%}
create table {{ relation }}
{{ create_statement_string }} {{ relation }}
{% set contract_config = config.get('contract') %}
{% if contract_config.enforced %}
{{ get_assert_columns_equivalent(sql) }}
Expand Down

This file was deleted.

0 comments on commit a855e39

Please sign in to comment.