diff --git a/dlt/destinations/impl/bigquery/bigquery.py b/dlt/destinations/impl/bigquery/bigquery.py index 2b3927e7c9..10a344f768 100644 --- a/dlt/destinations/impl/bigquery/bigquery.py +++ b/dlt/destinations/impl/bigquery/bigquery.py @@ -401,10 +401,7 @@ def _get_info_schema_columns_query( return query, folded_table_names def _get_column_def_sql(self, column: TColumnSchema, table: PreparedTableSchema = None) -> str: - name = self.sql_client.escape_column_name(column["name"]) - column_def_sql = ( - f"{name} {self.type_mapper.to_destination_type(column, table)} {self._gen_not_null(column.get('nullable', True))}" - ) + column_def_sql = super()._get_column_def_sql(column, table) if column.get(ROUND_HALF_EVEN_HINT, False): column_def_sql += " OPTIONS (rounding_mode='ROUND_HALF_EVEN')" if column.get(ROUND_HALF_AWAY_FROM_ZERO_HINT, False): diff --git a/dlt/destinations/impl/clickhouse/clickhouse.py b/dlt/destinations/impl/clickhouse/clickhouse.py index 3a5f5c3e28..a407e56361 100644 --- a/dlt/destinations/impl/clickhouse/clickhouse.py +++ b/dlt/destinations/impl/clickhouse/clickhouse.py @@ -292,11 +292,10 @@ def _get_table_update_sql( return sql - @staticmethod - def _gen_not_null(v: bool) -> str: + def _gen_not_null(self, v: bool) -> str: # ClickHouse fields are not nullable by default. # We use the `Nullable` modifier instead of NULL / NOT NULL modifiers to cater for ALTER statement. - pass + return "" def _from_db_type( self, ch_t: str, precision: Optional[int], scale: Optional[int] diff --git a/dlt/destinations/job_client_impl.py b/dlt/destinations/job_client_impl.py index 12cb129812..888c80c006 100644 --- a/dlt/destinations/job_client_impl.py +++ b/dlt/destinations/job_client_impl.py @@ -635,8 +635,7 @@ def _get_column_hints_sql(self, c: TColumnSchema) -> str: if c.get(h, False) is True # use ColumnPropInfos to get default value ) - @staticmethod - def _gen_not_null(nullable: bool) -> str: + def _gen_not_null(self, nullable: bool) -> str: return "NOT NULL" if not nullable else "" def _create_table_update( diff --git a/docs/website/docs/dlt-ecosystem/destinations/snowflake.md b/docs/website/docs/dlt-ecosystem/destinations/snowflake.md index 07cf822973..28684c39ac 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/snowflake.md +++ b/docs/website/docs/dlt-ecosystem/destinations/snowflake.md @@ -200,6 +200,12 @@ Note that we ignore missing columns `ERROR_ON_COLUMN_COUNT_MISMATCH = FALSE` and ## Supported column hints Snowflake supports the following [column hints](../../general-usage/schema#tables-and-columns): * `cluster` - Creates a cluster column(s). Many columns per table are supported and only when a new table is created. +* `unique` - Creates UNIQUE hint on a Snowflake column, can be added to many columns. ([optional](#additional-destination-options)) +* `primary_key` - Creates PRIMARY KEY on selected column(s), may be compound. ([optional](#additional-destination-options)) + +`unique` and `primary_key` are not enforced and `dlt` does not instruct Snowflake to `RELY` on them when +query planning. + ## Table and column identifiers Snowflake supports both case-sensitive and case-insensitive identifiers. All unquoted and uppercase identifiers resolve case-insensitively in SQL statements. Case-insensitive [naming conventions](../../general-usage/naming-convention.md#case-sensitive-and-insensitive-destinations) like the default **snake_case** will generate case-insensitive identifiers. Case-sensitive (like **sql_cs_v1**) will generate @@ -308,6 +314,7 @@ pipeline = dlt.pipeline( ## Additional destination options You can define your own stage to PUT files and disable the removal of the staged files after loading. +You can also opt-in to [create indexes](#supported-column-hints). ```toml [destination.snowflake] @@ -315,6 +322,8 @@ You can define your own stage to PUT files and disable the removal of the staged stage_name="DLT_STAGE" # Whether to keep or delete the staged files after COPY INTO succeeds keep_staged_files=true +# Add UNIQUE and PRIMARY KEY hints to tables +create_indexes=true ``` ### Setting up CSV format diff --git a/tests/load/dremio/test_dremio_client.py b/tests/load/dremio/test_dremio_client.py index efc72c0652..98212efb13 100644 --- a/tests/load/dremio/test_dremio_client.py +++ b/tests/load/dremio/test_dremio_client.py @@ -48,12 +48,12 @@ def test_dremio_factory() -> None: [ TColumnSchema(name="foo", data_type="text", partition=True), TColumnSchema(name="bar", data_type="bigint", sort=True), - TColumnSchema(name="baz", data_type="double"), + TColumnSchema(name="baz", data_type="double", nullable=False), ], False, [ 'CREATE TABLE "test_database"."test_dataset"."event_test_table"' - ' (\n"foo" VARCHAR ,\n"bar" BIGINT ,\n"baz" DOUBLE )\nPARTITION BY' + ' (\n"foo" VARCHAR ,\n"bar" BIGINT ,\n"baz" DOUBLE NOT NULL)\nPARTITION BY' ' ("foo")\nLOCALSORT BY ("bar")' ], ), @@ -66,7 +66,7 @@ def test_dremio_factory() -> None: False, [ 'CREATE TABLE "test_database"."test_dataset"."event_test_table"' - ' (\n"foo" VARCHAR ,\n"bar" BIGINT ,\n"baz" DOUBLE )\nPARTITION BY' + ' (\n"foo" VARCHAR ,\n"bar" BIGINT ,\n"baz" DOUBLE )\nPARTITION BY' ' ("foo","bar")' ], ), @@ -79,7 +79,7 @@ def test_dremio_factory() -> None: False, [ 'CREATE TABLE "test_database"."test_dataset"."event_test_table"' - ' (\n"foo" VARCHAR ,\n"bar" BIGINT ,\n"baz" DOUBLE )' + ' (\n"foo" VARCHAR ,\n"bar" BIGINT ,\n"baz" DOUBLE )' ], ), ],