From 170d977a12f402f12738593f5a3ef40045ed2a8b Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Fri, 10 Nov 2023 16:32:54 +0900 Subject: [PATCH 1/2] Remove unnecessary extra_py_files for Delta Lake --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 87b7ae13..addeb58a 100644 --- a/README.md +++ b/README.md @@ -479,8 +479,6 @@ You can also use Delta Lake to be able to use merge feature on tables. - To add the following config in your Interactive Session Config (in your profile): `conf: "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog` **Athena:** Athena is not compatible by default with delta tables, but you can configure the adapter to create Athena tables on top of your delta table. To do so, you need to configure the two following options in your profile: -- For Delta Lake 2.1.0 supported natively in Glue 4.0: `extra_py_files: "/opt/aws_glue_connectors/selected/datalake/delta-core_2.12-2.1.0.jar"` -- For Delta Lake 1.0.0 supported natively in Glue 3.0: `extra_py_files: "/opt/aws_glue_connectors/selected/datalake/delta-core_2.12-1.0.0.jar"` - `delta_athena_prefix: "the_prefix_of_your_choice"` - If your table is partitioned, then the add of new partition is not automatic, you need to perform an `MSCK REPAIR TABLE your_delta_table` after each new partition adding @@ -502,7 +500,6 @@ test_project: location: "s3://aws-dbt-glue-datalake-1234567890-eu-west-1/" datalake_formats: delta conf: "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" - extra_py_files: "/opt/aws_glue_connectors/selected/datalake/delta-core_2.12-2.1.0.jar" delta_athena_prefix: "delta" ``` From fc459308cdd124a4b9f300abaa70b54a7cfb3f42 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Fri, 10 Nov 2023 17:03:24 +0900 Subject: [PATCH 2/2] Update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a53670cf..aa2a7b00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## next version +- Remove unnecessary parameter for Delta Lake from readme + ## v1.7.0 - add compatibility with dbt 1.6 - fixed tests