Skip to content

Commit

Permalink
add support for name and desc
Browse files Browse the repository at this point in the history
  • Loading branch information
ion-elgreco committed Nov 18, 2023
1 parent 5d7ab06 commit 95078dd
Show file tree
Hide file tree
Showing 5 changed files with 63 additions and 9 deletions.
47 changes: 47 additions & 0 deletions crates/deltalake-core/src/operations/write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,12 @@ pub struct WriteBuilder {
writer_properties: Option<WriterProperties>,
/// Additional metadata to be added to commit
app_metadata: Option<HashMap<String, serde_json::Value>>,
/// Name of the table, only used when table doesn't exist yet
name: Option<String>,
/// Description of the table, only used when table doesn't exist yet
description: Option<String>,
/// Configurations of the delta table, only used when table doesn't exist
// configuration: Option<HashMap<String, Option<String>>>,
}

impl WriteBuilder {
Expand All @@ -131,6 +137,9 @@ impl WriteBuilder {
overwrite_schema: false,
writer_properties: None,
app_metadata: None,
name: None,
description: None,
// configuration: None,
}
}

Expand Down Expand Up @@ -214,6 +223,31 @@ impl WriteBuilder {
self
}

/// Specify the table name. Optionally qualified with
/// a database name [database_name.] table_name.
pub fn with_table_name(mut self, name: impl Into<String>) -> Self {
self.name = Some(name.into());
self
}

/// Comment to describe the table.
pub fn with_description(mut self, description: impl Into<String>) -> Self {
self.description = Some(description.into());
self
}

// /// Set configuration on created table
// pub fn with_configuration(
// mut self,
// configuration: impl IntoIterator<Item = (impl Into<String>, Option<impl Into<String>>)>,
// ) -> Self {
// self.configuration = configuration
// .into_iter()
// .map(|(k, v)| (k.into(), v.map(|s| s.into())))
// .collect();
// self
// }

async fn check_preconditions(&self) -> DeltaResult<Vec<Action>> {
match self.log_store.is_delta_table_location().await? {
true => {
Expand Down Expand Up @@ -242,6 +276,19 @@ impl WriteBuilder {
if let Some(partition_columns) = self.partition_columns.as_ref() {
builder = builder.with_partition_columns(partition_columns.clone())
}

if let Some(name) = self.name.as_ref() {
builder = builder.with_table_name(name.clone());
};

if let Some(desc) = self.description.as_ref() {
builder = builder.with_comment(desc.clone());
};

// if let Some(config) = self.configuration.as_ref() {
// builder = builder.with_configuration(config.clone());
// };

let (_, actions, _) = builder.into_table_and_actions()?;
Ok(actions)
}
Expand Down
4 changes: 2 additions & 2 deletions python/deltalake/_internal.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ def write_to_deltalake(
mode: str,
max_rows_per_group: int,
overwrite_schema: bool,
_name: Optional[str],
_description: Optional[str],
name: Optional[str],
description: Optional[str],
_configuration: Optional[Mapping[str, Optional[str]]],
storage_options: Optional[Dict[str, str]],
) -> None: ...
Expand Down
4 changes: 2 additions & 2 deletions python/deltalake/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,8 +214,8 @@ def write_deltalake(
mode=mode,
max_rows_per_group=max_rows_per_group,
overwrite_schema=overwrite_schema,
_name=name,
_description=description,
name=name,
description=description,
_configuration=configuration,
storage_options=storage_options,
)
Expand Down
12 changes: 10 additions & 2 deletions python/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1144,8 +1144,8 @@ fn write_to_deltalake(
max_rows_per_group: i64,
overwrite_schema: bool,
partition_by: Option<Vec<String>>,
_name: Option<String>,
_description: Option<String>,
name: Option<String>,
description: Option<String>,
_configuration: Option<HashMap<String, Option<String>>>,
storage_options: Option<HashMap<String, String>>,
) -> PyResult<()> {
Expand All @@ -1169,6 +1169,14 @@ fn write_to_deltalake(
builder = builder.with_partition_columns(partition_columns);
}

if let Some(name) = &name {
builder = builder.with_table_name(name);
};

if let Some(description) = &description {
builder = builder.with_description(description);
};

rt()?
.block_on(builder.into_future())
.map_err(PythonError::from)?;
Expand Down
5 changes: 2 additions & 3 deletions python/tests/test_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,7 @@ def test_local_path(
assert table == sample_data


@pytest.mark.skip(reason="Waiting on support with create matadata during write")
def test_roundtrip_metadata_rust(tmp_path: pathlib.Path, sample_data: pa.Table, engine):
def test_roundtrip_metadata_rust(tmp_path: pathlib.Path, sample_data: pa.Table):
write_deltalake(
tmp_path,
sample_data,
Expand All @@ -206,7 +205,7 @@ def test_roundtrip_metadata_rust(tmp_path: pathlib.Path, sample_data: pa.Table,

assert metadata.name == "test_name"
assert metadata.description == "test_desc"
assert metadata.configuration == {"delta.appendOnly": "false", "foo": "bar"}
# assert metadata.configuration == {"delta.appendOnly": "false", "foo": "bar"}


def test_roundtrip_metadata(tmp_path: pathlib.Path, sample_data: pa.Table):
Expand Down

0 comments on commit 95078dd

Please sign in to comment.