From e94df119cad6cb7d3744a402c42bf2f5204034fb Mon Sep 17 00:00:00 2001 From: jnadal Date: Tue, 6 Aug 2024 18:54:10 +0200 Subject: [PATCH] strip unwanted partition columns --- CHANGELOG.md | 3 ++- dbt/adapters/glue/impl.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eddb301..dec1e08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ -## New versio +## New version - Fix session provisioning timeout and delay handling +- Fix get_columns_in_relation function to stop returning additional partition columns ## v1.8.1 - Fix typo in README.md diff --git a/dbt/adapters/glue/impl.py b/dbt/adapters/glue/impl.py index fac5774..16562d8 100644 --- a/dbt/adapters/glue/impl.py +++ b/dbt/adapters/glue/impl.py @@ -265,6 +265,10 @@ def get_columns_in_relation(self, relation: BaseRelation): columns = [x for x in columns if x.name not in self.HUDI_METADATA_COLUMNS] + # strip partition columns. + columns = [x for x in columns + if not re.match(r'^Part \d+$', x.name)] + logger.debug("columns after strip:") logger.debug(columns)