Skip to content

Commit

Permalink
only overwrite schema variable if schema is none
Browse files Browse the repository at this point in the history
  • Loading branch information
ion-elgreco committed Nov 7, 2023
1 parent 26b9f1f commit 3b9eebc
Showing 1 changed file with 10 additions and 5 deletions.
15 changes: 10 additions & 5 deletions python/deltalake/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,18 +171,20 @@ def write_deltalake(
table.update_incremental()

if isinstance(data, RecordBatchReader):
data, schema = convert_pyarrow_recordbatchreader(data, large_dtypes)
data, delta_schema = convert_pyarrow_recordbatchreader(data, large_dtypes)
elif isinstance(data, pa.RecordBatch):
data, schema = convert_pyarrow_recordbatch(data, large_dtypes)
data, delta_schema = convert_pyarrow_recordbatch(data, large_dtypes)
elif isinstance(data, pa.Table):
data, schema = convert_pyarrow_table(data, large_dtypes)
data, delta_schema = convert_pyarrow_table(data, large_dtypes)
elif isinstance(data, ds.Dataset):
data, schema = convert_pyarrow_table(data.to_table(), large_dtypes)
data, delta_schema = convert_pyarrow_table(data.to_table(), large_dtypes)
elif _has_pandas and isinstance(data, pd.DataFrame):
if schema is not None:
data = pa.Table.from_pandas(data, schema=schema)
else:
data, schema = convert_pyarrow_table(pa.Table.from_pandas(data), False)
data, delta_schema = convert_pyarrow_table(
pa.Table.from_pandas(data), False
)
elif isinstance(data, Iterable):
if schema is None:
raise ValueError("You must provide schema if data is Iterable")
Expand All @@ -191,6 +193,9 @@ def write_deltalake(
f"{type(data).__name__} is not a valid input. Only PyArrow RecordBatchReader, RecordBatch, Iterable[RecordBatch], Table, Dataset or Pandas DataFrame are valid inputs for source."
)

if schema is None:
schema = delta_schema

if filesystem is not None:
raise NotImplementedError("Filesystem support is not yet implemented. #570")

Expand Down

0 comments on commit 3b9eebc

Please sign in to comment.