diff --git a/.changes/unreleased/Features-20241126-215421.yaml b/.changes/unreleased/Features-20241126-215421.yaml new file mode 100644 index 000000000..00020fb27 --- /dev/null +++ b/.changes/unreleased/Features-20241126-215421.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Adds Iceberg support as a new table format configuration +time: 2024-11-26T21:54:21.990317Z +custom: + Author: borjavb + Issue: "1370" diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index f166e5d05..949887052 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -3,7 +3,7 @@ {%- set raw_partition_by = config.get('partition_by', none) -%} {%- set raw_cluster_by = config.get('cluster_by', none) -%} {%- set sql_header = config.get('sql_header', none) -%} - + {%- set table_format = config.get('table_format', 'default') -%} {%- set partition_config = adapter.parse_partition_by(raw_partition_by) -%} {%- if partition_config.time_ingestion_partitioning -%} {%- set columns = get_columns_with_types_in_query_sql(sql) -%} @@ -23,10 +23,19 @@ {#-- cannot do contracts at the same time as time ingestion partitioning -#} {{ columns }} {% endif %} - {{ partition_by(partition_config) }} + {%- if table_format == "iceberg" and partition_config is not none-%} + {#-- Nov 2024. Limitations: PARTITION BY cannot be used in iceberg-#} + {% do exceptions.raise_compiler_error("Partition by not yet available in iceberg tables, use cluster by instead") %} + {%- else -%} + {{ partition_by(partition_config) }} + {% endif %} + {{ cluster_by(raw_cluster_by) }} - {{ bigquery_table_options(config, model, temporary) }} + {% if table_format == "iceberg" %} + {{ bigquery_iceberg_connection(config) }} + {{ bigquery_iceberg_table_options(config, relation) }} + {% endif %} {#-- PARTITION BY cannot be used with the AS query_statement clause. https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#partition_expression diff --git a/dbt/include/bigquery/macros/relations/table/options.sql b/dbt/include/bigquery/macros/relations/table/options.sql index 9f9b6b6d1..f458cff23 100644 --- a/dbt/include/bigquery/macros/relations/table/options.sql +++ b/dbt/include/bigquery/macros/relations/table/options.sql @@ -2,3 +2,25 @@ {% set opts = adapter.get_table_options(config, node, temporary) %} {%- do return(bigquery_options(opts)) -%} {%- endmacro -%} + +{% macro bigquery_iceberg_table_options(config, relation) %} + {% set base_location = config.get('base_location') %} + {%- if not base_location-%} + {% do exceptions.raise_compiler_error("base_location not found") %} + {% endif %} + {% set sub_path = relation.identifier %} + {% set storage_uri = base_location~'/'~sub_path %} + {% set opts = {'file_format':'"parquet"', + 'table_format':'"iceberg"', + 'storage_uri':'"'~storage_uri~'"' } + %} + {%- do return(bigquery_options(opts)) -%} +{%- endmacro -%} + +{% macro bigquery_iceberg_connection(config) %} + {% set connection = config.get('bl_connection') %} + {%- if not connection-%} + {% do exceptions.raise_compiler_error("BigLake connection not found") %} + {% endif %} + {%- do return("WITH CONNECTION `"~connection~"`") %} +{%- endmacro -%}