diff --git a/iceberg-rust-spec/src/spec/materialized_view_metadata.rs b/iceberg-rust-spec/src/spec/materialized_view_metadata.rs index 4209f0cf..f158bf89 100644 --- a/iceberg-rust-spec/src/spec/materialized_view_metadata.rs +++ b/iceberg-rust-spec/src/spec/materialized_view_metadata.rs @@ -75,7 +75,8 @@ mod tests { "location" : "s3://bucket/warehouse/default.db/event_agg", "current-version-id" : 1, "properties" : { - "comment" : "Daily event counts" + "comment" : "Daily event counts", + "storage_table": "s3://bucket/path/to/metadata.json" }, "versions" : [ { "version-id" : 1, @@ -113,8 +114,7 @@ mod tests { "version-log" : [ { "timestamp-ms" : 1573518431292, "version-id" : 1 - } ], - "materialization": "s3://bucket/path/to/metadata.json" + } ] } "#; let metadata = serde_json::from_str::(data) diff --git a/iceberg-rust-spec/src/spec/view_metadata.rs b/iceberg-rust-spec/src/spec/view_metadata.rs index 0398773c..662df72d 100644 --- a/iceberg-rust-spec/src/spec/view_metadata.rs +++ b/iceberg-rust-spec/src/spec/view_metadata.rs @@ -4,6 +4,7 @@ use std::{ collections::HashMap, + ops::{Deref, DerefMut}, time::{SystemTime, UNIX_EPOCH}, }; @@ -29,7 +30,7 @@ pub type ViewMetadataBuilder = GeneralViewMetadataBuilder>; #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default, Builder)] #[serde(try_from = "ViewMetadataEnum", into = "ViewMetadataEnum")] /// Fields for the version 1 of the view metadata. -pub struct GeneralViewMetadata { +pub struct GeneralViewMetadata { #[builder(default = "Uuid::new_v4()")] /// A UUID that identifies the view, generated when the view is created. Implementations must throw an exception if a view’s UUID does not match the expected UUID after refreshing metadata pub view_uuid: Uuid, @@ -54,11 +55,10 @@ pub struct GeneralViewMetadata { #[builder(default)] /// A string to string map of view properties. This is used for metadata such as “comment” and for settings that affect view maintenance. /// This is not intended to be used for arbitrary metadata. - pub properties: HashMap, - pub materialization: T, + pub properties: ViewProperties, } -impl GeneralViewMetadata { +impl GeneralViewMetadata { /// Get current schema #[inline] pub fn current_schema(&self, branch: Option<&str>) -> Result<&Schema, Error> { @@ -112,12 +112,12 @@ mod _serde { spec::{schema::SchemaV2, table_metadata::VersionNumber}, }; - use super::{FormatVersion, GeneralViewMetadata, Version, VersionLogStruct}; + use super::{FormatVersion, GeneralViewMetadata, Version, VersionLogStruct, ViewProperties}; /// Metadata of an iceberg view #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(untagged)] - pub(super) enum ViewMetadataEnum { + pub(super) enum ViewMetadataEnum { /// Version 1 of the table metadata V1(ViewMetadataV1), } @@ -125,7 +125,7 @@ mod _serde { #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] /// Fields for the version 1 of the view metadata. - pub struct ViewMetadataV1 { + pub struct ViewMetadataV1 { /// A UUID that identifies the view, generated when the view is created. Implementations must throw an exception if a view’s UUID does not match the expected UUID after refreshing metadata pub view_uuid: Uuid, /// An integer version number for the view format; must be 1 @@ -143,11 +143,10 @@ mod _serde { pub schemas: Vec, /// A string to string map of view properties. This is used for metadata such as “comment” and for settings that affect view maintenance. /// This is not intended to be used for arbitrary metadata. - pub properties: Option>, - pub materialization: T, + pub properties: Option>, } - impl TryFrom> for GeneralViewMetadata { + impl TryFrom> for GeneralViewMetadata { type Error = Error; fn try_from(value: ViewMetadataEnum) -> Result { match value { @@ -156,7 +155,7 @@ mod _serde { } } - impl From> for ViewMetadataEnum { + impl From> for ViewMetadataEnum { fn from(value: GeneralViewMetadata) -> Self { match value.format_version { FormatVersion::V1 => ViewMetadataEnum::V1(value.into()), @@ -164,7 +163,7 @@ mod _serde { } } - impl TryFrom> for GeneralViewMetadata { + impl TryFrom> for GeneralViewMetadata { type Error = Error; fn try_from(value: ViewMetadataV1) -> Result { Ok(GeneralViewMetadata { @@ -175,7 +174,6 @@ mod _serde { versions: HashMap::from_iter(value.versions.into_iter().map(|x| (x.version_id, x))), version_log: value.version_log, properties: value.properties.unwrap_or_default(), - materialization: value.materialization, schemas: HashMap::from_iter( value .schemas @@ -187,7 +185,7 @@ mod _serde { } } - impl From> for ViewMetadataV1 { + impl From> for ViewMetadataV1 { fn from(value: GeneralViewMetadata) -> Self { ViewMetadataV1 { view_uuid: value.view_uuid, @@ -201,12 +199,32 @@ mod _serde { } else { Some(value.properties) }, - materialization: value.materialization, schemas: value.schemas.into_values().map(Into::into).collect(), } } } } + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default)] +pub struct ViewProperties { + pub storage_table: T, + #[serde(flatten)] + pub other: HashMap, +} + +impl Deref for ViewProperties { + type Target = HashMap; + fn deref(&self) -> &Self::Target { + &self.other + } +} + +impl DerefMut for ViewProperties { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.other + } +} + #[derive(Debug, Serialize_repr, Deserialize_repr, PartialEq, Eq, Clone)] #[repr(u8)] /// Iceberg format version diff --git a/iceberg-rust/src/catalog/commit.rs b/iceberg-rust/src/catalog/commit.rs index 626b3d67..7417d0af 100644 --- a/iceberg-rust/src/catalog/commit.rs +++ b/iceberg-rust/src/catalog/commit.rs @@ -288,14 +288,15 @@ pub fn check_table_requirements( } /// Check table update requirements -pub fn check_view_requirements( +pub fn check_view_requirements( requirements: &[ViewRequirement], metadata: &GeneralViewMetadata, ) -> bool { requirements.iter().all(|x| match x { ViewRequirement::AssertViewUuid { uuid } => metadata.view_uuid == *uuid, ViewRequirement::AssertMaterialization { materialization } => { - metadata.materialization == *(materialization as &dyn Any).downcast_ref::().unwrap() + metadata.properties.storage_table + == *(materialization as &dyn Any).downcast_ref::().unwrap() } }) } @@ -373,7 +374,7 @@ pub fn apply_table_updates( } /// Apply updates to metadata -pub fn apply_view_updates( +pub fn apply_view_updates( metadata: &mut GeneralViewMetadata, updates: Vec, ) -> Result<(), Error> { @@ -411,7 +412,7 @@ pub fn apply_view_updates( metadata.current_version_id = view_version_id; } ViewUpdate::SetMaterialization { materialization } => { - metadata.materialization = (&materialization as &dyn Any) + metadata.properties.storage_table = (&materialization as &dyn Any) .downcast_ref::() .cloned() .ok_or(Error::InvalidFormat( diff --git a/iceberg-rust/src/materialized_view/materialized_view_builder.rs b/iceberg-rust/src/materialized_view/materialized_view_builder.rs index 9dd437e9..e46bdcc4 100644 --- a/iceberg-rust/src/materialized_view/materialized_view_builder.rs +++ b/iceberg-rust/src/materialized_view/materialized_view_builder.rs @@ -13,7 +13,7 @@ use iceberg_rust_spec::{ materialized_view_metadata::MaterializedViewMetadataBuilder, schema::Schema, table_metadata::TableMetadataBuilder, - view_metadata::{VersionBuilder, ViewRepresentation, REF_PREFIX}, + view_metadata::{VersionBuilder, ViewProperties, ViewRepresentation, REF_PREFIX}, }, util::strip_prefix, }; @@ -68,12 +68,11 @@ impl MaterializedViewBuilder { .schema_id(1) .build()?, )) - .materialization("".to_owned()) .current_version_id(1) - .properties(HashMap::from_iter(vec![( - REF_PREFIX.to_string() + "main", - 1.to_string(), - )])); + .properties(ViewProperties { + storage_table: "".to_owned(), + other: HashMap::from_iter(vec![(REF_PREFIX.to_string() + "main", 1.to_string())]), + }); Ok(Self { identifier: Identifier::parse(&identifier.to_string())?, catalog, @@ -106,7 +105,7 @@ impl MaterializedViewBuilder { + "-" + &Uuid::new_v4().to_string() + ".metadata.json"; - metadata.materialization = table_path.clone(); + metadata.properties.storage_table = table_path.clone(); let table_metadata_json = serde_json::to_string(&table_metadata)?; object_store .put( diff --git a/iceberg-rust/src/materialized_view/mod.rs b/iceberg-rust/src/materialized_view/mod.rs index f82a2363..44daf258 100644 --- a/iceberg-rust/src/materialized_view/mod.rs +++ b/iceberg-rust/src/materialized_view/mod.rs @@ -87,7 +87,7 @@ impl MaterializedView { } /// Get the storage table of the materialized view pub async fn storage_table(&self) -> Result { - let storage_table_location = &self.metadata.materialization; + let storage_table_location = &self.metadata.properties.storage_table; let bucket = parse_bucket(storage_table_location)?; if let TabularMetadata::Table(metadata) = serde_json::from_str(std::str::from_utf8( &self diff --git a/iceberg-rust/src/view/transaction/operation.rs b/iceberg-rust/src/view/transaction/operation.rs index 5ffb41a7..d8598c87 100644 --- a/iceberg-rust/src/view/transaction/operation.rs +++ b/iceberg-rust/src/view/transaction/operation.rs @@ -35,7 +35,7 @@ pub enum Operation { UpdateMaterialization(T), } -impl Operation { +impl Operation { /// Execute operation pub async fn execute( self, @@ -105,7 +105,7 @@ impl Operation { )), Operation::UpdateMaterialization(materialization) => { let previous_materialization = - (&metadata.materialization as &dyn Any).downcast_ref::(); + (&metadata.properties.storage_table as &dyn Any).downcast_ref::(); let materialization = (&materialization as &dyn Any) .downcast_ref::() .ok_or(Error::InvalidFormat( diff --git a/iceberg-rust/src/view/view_builder.rs b/iceberg-rust/src/view/view_builder.rs index 3e5ac1bd..acc73165 100644 --- a/iceberg-rust/src/view/view_builder.rs +++ b/iceberg-rust/src/view/view_builder.rs @@ -11,7 +11,7 @@ use crate::catalog::tabular::Tabular; use crate::error::Error; use iceberg_rust_spec::spec::schema::Schema; use iceberg_rust_spec::spec::view_metadata::{ - VersionBuilder, ViewMetadataBuilder, ViewRepresentation, REF_PREFIX, + VersionBuilder, ViewMetadataBuilder, ViewProperties, ViewRepresentation, REF_PREFIX, }; use super::Catalog; @@ -60,12 +60,11 @@ impl ViewBuilder { .schema_id(1) .build()?, )) - .materialization(None) .current_version_id(1) - .properties(HashMap::from_iter(vec![( - REF_PREFIX.to_string() + "main", - 1.to_string(), - )])); + .properties(ViewProperties { + storage_table: None, + other: HashMap::from_iter(vec![(REF_PREFIX.to_string() + "main", 1.to_string())]), + }); Ok(ViewBuilder { metadata: builder, identifier,