write with non compliant types

delta-io · Nov 4, 2023 · 4d4de1f · 4d4de1f
1 parent e70770b
commit 4d4de1f
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/python/deltalake/writer.py b/python/deltalake/writer.py
@@ -302,6 +302,10 @@ def validate_batch(batch: pa.RecordBatch) -> pa.RecordBatch:
             schema, (validate_batch(batch) for batch in batch_iter)
         )
 
+    if file_options is not None:
+        file_options.update(use_compliant_nested_type=False)
+    else:
+        file_options = pa.dataset.ParquetFileFormat().make_write_options(use_compliant_nested_type=False)
     ds.write_dataset(
         data,
         base_dir="/",