Merge pull request #15 from JanKaul/fix-metadata

Fix metadata
JanKaul · Jan 22, 2024 · a67f664 · a67f664
2 parents 004589a + 4a9616d
commit a67f664
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 6 deletions.
diff --git a/iceberg-rust/src/table/mod.rs b/iceberg-rust/src/table/mod.rs
@@ -12,7 +12,7 @@ use iceberg_rust_spec::spec::{
     manifest_list::ManifestListEntry,
     schema::Schema,
     snapshot::{Operation, Snapshot, SnapshotReference, SnapshotRetention, Summary},
-    table_metadata::{TableMetadata, MAIN_BRANCH},
+    table_metadata::{SnapshotLog, TableMetadata, MAIN_BRANCH},
 };
 use iceberg_rust_spec::util::{self, strip_prefix};
 
@@ -272,14 +272,15 @@ impl Table {
             + &snapshot_id.to_string()
             + &uuid::Uuid::new_v4().to_string()
             + ".avro";
+        let timestamp_ms = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_millis() as i64;
         let snapshot = Snapshot {
             snapshot_id,
             parent_snapshot_id,
             sequence_number: metadata.last_sequence_number + 1,
-            timestamp_ms: SystemTime::now()
-                .duration_since(SystemTime::UNIX_EPOCH)
-                .unwrap()
-                .as_millis() as i64,
+            timestamp_ms,
             manifest_list: new_manifest_list_location,
             summary: Summary {
                 operation: Operation::Append,
@@ -294,6 +295,10 @@ impl Table {
         if branch_name == MAIN_BRANCH {
             metadata.current_snapshot_id = Some(snapshot_id);
         }
+        metadata.snapshot_log.push(SnapshotLog {
+            snapshot_id,
+            timestamp_ms,
+        });
         metadata
             .refs
             .entry(branch_name)

diff --git a/iceberg-rust/src/table/table_builder.rs b/iceberg-rust/src/table/table_builder.rs
@@ -47,7 +47,23 @@ impl TableBuilder {
     }
     /// Building a table writes the metadata file and commits the table to either the metastore or the filesystem
     pub async fn build(&mut self) -> Result<Table, Error> {
-        let metadata = self.metadata.build()?;
+        let mut metadata = self.metadata.build()?;
+        let last_column_id = metadata
+            .schemas
+            .values()
+            .flat_map(|x| x.fields.fields.iter())
+            .map(|x| x.id)
+            .max()
+            .unwrap_or(0);
+        metadata.last_column_id = last_column_id;
+        let last_column_id = metadata
+            .partition_specs
+            .values()
+            .flat_map(|x| x.fields.iter())
+            .map(|x| x.field_id)
+            .max()
+            .unwrap_or(0);
+        metadata.last_partition_id = last_column_id;
         let bucket = parse_bucket(&metadata.location)?;
         let object_store = self.catalog.object_store(bucket);