implement choice of delimiter for seed files (#1122) (#1380)

* implement choice of delimiter for seed files * adding change log entry * implementation of test of TestBigQuerySeedWithUniqueDelimiter * Update dbt/adapters/bigquery/impl.py Co-authored-by: Doug Beatty <[email protected]> * Update dbt/include/bigquery/macros/materializations/seed.sql Co-authored-by: Doug Beatty <[email protected]> * Update dbt/include/bigquery/macros/materializations/seed.sql Co-authored-by: Doug Beatty <[email protected]> * Update dbt/adapters/bigquery/impl.py Co-authored-by: Doug Beatty <[email protected]> * Update .changes/unreleased/Fixes-20240226-233024.yaml --------- Co-authored-by: Doug Beatty <[email protected]> Co-authored-by: Colin Rogers <[email protected]> (cherry picked from commit 0d37ba6) Co-authored-by: salimmoulouel <[email protected]>
dbt-labs · Oct 22, 2024 · 1ae695b · 1ae695b
1 parent 2180bf8
commit 1ae695b
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 4 deletions.
diff --git a/.changes/unreleased/Fixes-20240226-233024.yaml b/.changes/unreleased/Fixes-20240226-233024.yaml
@@ -0,0 +1,6 @@
+kind: Fixes
+body: make seed delimiter configurable via `field_delimeter` in model config
+time: 2024-02-26T23:30:24.141213+01:00
+custom:
+  Author: salimmoulouel
+  Issue: "1119"
diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py
@@ -654,7 +654,9 @@ def alter_table_add_columns(self, relation, columns):
         client.update_table(new_table, ["schema"])
 
     @available.parse_none
-    def load_dataframe(self, database, schema, table_name, agate_table, column_override):
+    def load_dataframe(
+        self, database, schema, table_name, agate_table, column_override, field_delimiter
+    ):
         bq_schema = self._agate_to_schema(agate_table, column_override)
         conn = self.connections.get_thread_connection()
         client = conn.handle
@@ -664,7 +666,7 @@ def load_dataframe(self, database, schema, table_name, agate_table, column_overr
         load_config = google.cloud.bigquery.LoadJobConfig()
         load_config.skip_leading_rows = 1
         load_config.schema = bq_schema
-
+        load_config.field_delimiter = field_delimiter
         with open(agate_table.original_abspath, "rb") as f:
             job = client.load_table_from_file(f, table_ref, rewind=True, job_config=load_config)
 

diff --git a/dbt/include/bigquery/macros/materializations/seed.sql b/dbt/include/bigquery/macros/materializations/seed.sql
@@ -11,7 +11,7 @@
 
   {%- set column_override = model['config'].get('column_types', {}) -%}
   {{ adapter.load_dataframe(model['database'], model['schema'], model['alias'],
-  							agate_table, column_override) }}
+  							agate_table, column_override, model['config']['delimiter']) }}
 
   {% call statement() %}
     alter table {{ this.render() }} set {{ bigquery_table_options(config, model) }}

diff --git a/tests/functional/adapter/test_simple_seed.py b/tests/functional/adapter/test_simple_seed.py
@@ -5,7 +5,6 @@
 from dbt.tests.adapter.simple_seed.test_seed import BaseTestEmptySeed
 from dbt.tests.adapter.utils.base_utils import run_dbt
 
-
 _SEED_CONFIGS_CSV = """
 seed_id,stuff
 1,a
@@ -156,3 +155,38 @@ def test__bigquery_seed_table_with_labels_config_bigquery(self, project):
 
 class TestBigQueryEmptySeed(BaseTestEmptySeed):
     pass
+
+
+class TestBigQuerySeedWithUniqueDelimiter(TestSimpleSeedConfigs):
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "seed_enabled.csv": seeds__enabled_in_config_csv.replace(",", "|"),
+            "seed_tricky.csv": seeds__tricky_csv.replace(",", "\t"),
+            "seed_configs.csv": _SEED_CONFIGS_CSV,
+        }
+
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "config-version": 2,
+            "seeds": {
+                "test": {
+                    "enabled": False,
+                    "quote_columns": True,
+                    "seed_enabled": {
+                        "enabled": True,
+                        "+column_types": self.seed_enabled_types(),
+                        "delimiter": "|",
+                    },
+                    "seed_tricky": {
+                        "enabled": True,
+                        "+column_types": self.seed_tricky_types(),
+                        "delimiter": "\t",
+                    },
+                    "seed_configs": {
+                        "enabled": True,
+                    },
+                },
+            },
+        }