From 166817c20c9c6eaeec91bc4c94089a6a8bdf6478 Mon Sep 17 00:00:00 2001 From: Christophe Oudar <90898980+github-christophe-oudar@users.noreply.github.com> Date: Tue, 22 Aug 2023 20:27:11 +0200 Subject: [PATCH] case insensitive check on partition matching (#888) * case insensitive check on partition matching * Review change --------- Co-authored-by: Christophe Oudar --- .../unreleased/Fixes-20230818-214616.yaml | 6 ++++ dbt/adapters/bigquery/__init__.py | 2 +- dbt/adapters/bigquery/impl.py | 5 +-- tests/unit/test_bigquery_adapter.py | 31 ++++++++++++++++++- 4 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Fixes-20230818-214616.yaml diff --git a/.changes/unreleased/Fixes-20230818-214616.yaml b/.changes/unreleased/Fixes-20230818-214616.yaml new file mode 100644 index 000000000..a66059e21 --- /dev/null +++ b/.changes/unreleased/Fixes-20230818-214616.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: case insensitive check on partition matching +time: 2023-08-18T21:46:16.828488+02:00 +custom: + Author: Kayrnt + Issue: "886" diff --git a/dbt/adapters/bigquery/__init__.py b/dbt/adapters/bigquery/__init__.py index b66ef278a..adbd67590 100644 --- a/dbt/adapters/bigquery/__init__.py +++ b/dbt/adapters/bigquery/__init__.py @@ -2,7 +2,7 @@ from dbt.adapters.bigquery.connections import BigQueryCredentials from dbt.adapters.bigquery.relation import BigQueryRelation # noqa from dbt.adapters.bigquery.column import BigQueryColumn # noqa -from dbt.adapters.bigquery.impl import BigQueryAdapter, GrantTarget # noqa +from dbt.adapters.bigquery.impl import BigQueryAdapter, GrantTarget, PartitionConfig # noqa from dbt.adapters.base import AdapterPlugin from dbt.include import bigquery diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index f53cd4084..3f8351861 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -620,12 +620,13 @@ def _partitions_match(table, conf_partition: Optional[PartitionConfig]) -> bool: table_field = ( table.time_partitioning.field.lower() if table.time_partitioning.field else None ) + table_granularity = table.partitioning_type conf_table_field = conf_partition.field return ( - table_field == conf_table_field + table_field == conf_table_field.lower() or (conf_partition.time_ingestion_partitioning and table_field is not None) - ) and table_granularity == conf_partition.granularity + ) and table_granularity.lower() == conf_partition.granularity.lower() elif conf_partition and table.range_partitioning is not None: dest_part = table.range_partitioning conf_part = conf_partition.range or {} diff --git a/tests/unit/test_bigquery_adapter.py b/tests/unit/test_bigquery_adapter.py index 56b8e07d7..bb98db86f 100644 --- a/tests/unit/test_bigquery_adapter.py +++ b/tests/unit/test_bigquery_adapter.py @@ -12,10 +12,12 @@ import dbt.dataclass_schema +from dbt.adapters.bigquery import PartitionConfig from dbt.adapters.bigquery import BigQueryCredentials from dbt.adapters.bigquery import BigQueryAdapter from dbt.adapters.bigquery import BigQueryRelation from dbt.adapters.bigquery import Plugin as BigQueryPlugin +from google.cloud.bigquery.table import Table from dbt.adapters.bigquery.connections import BigQueryConnectionManager from dbt.adapters.bigquery.connections import _sanitize_label, _VALIDATE_LABEL_LENGTH_LIMIT from dbt.adapters.base.query_headers import MacroQueryStringSetter @@ -376,7 +378,10 @@ def test_location_user_agent(self, mock_bq, mock_auth_default): mock_client.assert_not_called() connection.handle mock_client.assert_called_once_with( - "dbt-unit-000000", creds, location="Luna Station", client_info=HasUserAgent() + "dbt-unit-000000", + creds, + location="Luna Station", + client_info=HasUserAgent(), ) @@ -1023,6 +1028,30 @@ def test_convert_time_type(self): for col_idx, expect in enumerate(expected): assert BigQueryAdapter.convert_time_type(agate_table, col_idx) == expect + # The casing in this case can't be enforced on the API side, + # so we have to validate that we have a case-insensitive comparison + def test_partitions_match(self): + table = Table.from_api_repr( + { + "tableReference": { + "projectId": "test-project", + "datasetId": "test_dataset", + "tableId": "test_table", + }, + "timePartitioning": {"type": "DAY", "field": "ts"}, + } + ) + partition_config = PartitionConfig.parse( + { + "field": "TS", + "data_type": "date", + "granularity": "day", + "time_ingestion_partitioning": False, + "copy_partitions": False, + } + ) + assert BigQueryAdapter._partitions_match(table, partition_config) is True + class TestBigQueryGrantAccessTo(BaseTestBigQueryAdapter): entity = BigQueryRelation.from_dict(