Move inside iceberg crate

Cargo.toml

@@ -23,7 +23,6 @@ members = [
     "crates/iceberg",
     "crates/integration_tests",
     "crates/integrations/*",
-    "crates/puffin",
     "crates/test_utils",
 ]
 exclude = ["bindings/python"]

crates/iceberg/Cargo.toml

@@ -81,6 +81,7 @@ tokio = { workspace = true, optional = true }
 typed-builder = { workspace = true }
 url = { workspace = true }
 uuid = { workspace = true }
+zstd = { workspace = true }
 
 [dev-dependencies]
 ctor = { workspace = true }

crates/iceberg/src/lib.rs

@@ -84,3 +84,5 @@ mod runtime;
 pub mod arrow;
 mod utils;
 pub mod writer;
+
+pub mod puffin;

crates/puffin/src/compression.rs → crates/iceberg/src/puffin/compression.rs

@@ -15,12 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use iceberg::{Error, ErrorKind, Result};
 use serde::{Deserialize, Serialize};
 
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone, Copy)]
+use crate::{Error, ErrorKind, Result};
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Default, Serialize, Deserialize)]
 #[serde(rename_all = "lowercase")]
-#[derive(Default)]
 /// Data compression formats
 pub enum CompressionCodec {
     #[default]
@@ -73,7 +73,7 @@ impl CompressionCodec {
 
 #[cfg(test)]
 mod tests {
-    use crate::compression::CompressionCodec;
+    use crate::puffin::compression::CompressionCodec;
 
     #[tokio::test]
     async fn test_compression_codec_none() {

crates/puffin/src/metadata.rs → crates/iceberg/src/puffin/metadata.rs

@@ -18,12 +18,12 @@
 use std::collections::{HashMap, HashSet};
 
 use bytes::Bytes;
-use iceberg::io::{FileRead, InputFile};
-use iceberg::{Error, ErrorKind, Result};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
-use crate::compression::CompressionCodec;
+use crate::io::{FileRead, InputFile};
+use crate::puffin::compression::CompressionCodec;
+use crate::{Error, ErrorKind, Result};
 
 /// Human-readable identification of the application writing the file, along with its version.
 /// Example: "Trino version 381"
@@ -286,11 +286,11 @@ mod tests {
     use std::collections::HashMap;
 
     use bytes::Bytes;
-    use iceberg::io::{FileIOBuilder, InputFile};
     use tempfile::TempDir;
 
-    use crate::metadata::{BlobMetadata, CompressionCodec, FileMetadata};
-    use crate::test_utils::{
+    use crate::io::{FileIOBuilder, InputFile};
+    use crate::puffin::metadata::{BlobMetadata, CompressionCodec, FileMetadata};
+    use crate::puffin::test_utils::{
         empty_footer_payload, empty_footer_payload_bytes, empty_footer_payload_bytes_length_bytes,
         rust_empty_uncompressed_input_file, rust_uncompressed_metric_input_file,
         rust_zstd_compressed_metric_input_file, uncompressed_metric_file_metadata,

crates/iceberg/src/puffin/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Iceberg Puffin file format implementation
+
+#![deny(missing_docs)]
+
+mod blob;
+pub use blob::{Blob, APACHE_DATASKETCHES_THETA_V1};
+
+mod compression;
+pub use compression::CompressionCodec;
+
+mod metadata;
+pub use metadata::{BlobMetadata, FileMetadata, CREATED_BY_PROPERTY};
+
+mod reader;
+pub use reader::PuffinReader;
+
+#[cfg(test)]
+mod test_utils;
+
+mod writer;
+pub use writer::PuffinWriter;

crates/puffin/src/reader.rs → crates/iceberg/src/puffin/reader.rs

@@ -15,11 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use iceberg::io::{FileRead, InputFile};
-use iceberg::Result;
-
-use crate::blob::Blob;
-use crate::metadata::{BlobMetadata, FileMetadata};
+use crate::io::{FileRead, InputFile};
+use crate::puffin::blob::Blob;
+use crate::puffin::metadata::{BlobMetadata, FileMetadata};
+use crate::Result;
 
 /// Puffin reader
 pub struct PuffinReader {
@@ -68,12 +67,12 @@ impl PuffinReader {
 #[cfg(test)]
 mod tests {
 
-    use crate::test_utils::{
+    use crate::puffin::test_utils::{
         blob_0, blob_1, rust_uncompressed_metric_input_file,
         rust_zstd_compressed_metric_input_file, uncompressed_metric_file_metadata,
         zstd_compressed_metric_file_metadata,
     };
-    use crate::PuffinReader;
+    use crate::puffin::PuffinReader;
 
     #[tokio::test]
     async fn test_puffin_reader_uncompressed_metric_data() {

crates/puffin/src/test_utils.rs → crates/iceberg/src/puffin/test_utils.rs

@@ -17,14 +17,13 @@
 
 use std::collections::HashMap;
 
-use iceberg::io::{FileIOBuilder, InputFile};
+use crate::io::{FileIOBuilder, InputFile};
+use crate::puffin::blob::Blob;
+use crate::puffin::compression::CompressionCodec;
+use crate::puffin::metadata::{BlobMetadata, FileMetadata, CREATED_BY_PROPERTY};
 
-use crate::blob::Blob;
-use crate::compression::CompressionCodec;
-use crate::metadata::{BlobMetadata, FileMetadata, CREATED_BY_PROPERTY};
-
-const V1_RUST: &str = "testdata/v1/rust-generated";
-const V1_JAVA: &str = "testdata/v1/java-generated";
+const RUST_TESTDATA: &str = "testdata/puffin/rust-generated";
+const JAVA_TESTDATA: &str = "testdata/puffin/java-generated";
 const EMPTY_UNCOMPRESSED: &str = "empty-puffin-uncompressed.bin";
 const METRIC_UNCOMPRESSED: &str = "sample-metric-data-uncompressed.bin";
 const METRIC_ZSTD_COMPRESSED: &str = "sample-metric-data-compressed-zstd.bin";
@@ -38,27 +37,27 @@ fn input_file_for_test_data(path: &str) -> InputFile {
 }
 
 pub(crate) fn java_empty_uncompressed_input_file() -> InputFile {
-    input_file_for_test_data(&[V1_JAVA, EMPTY_UNCOMPRESSED].join("/"))
+    input_file_for_test_data(&[JAVA_TESTDATA, EMPTY_UNCOMPRESSED].join("/"))
 }
 
 pub(crate) fn rust_empty_uncompressed_input_file() -> InputFile {
-    input_file_for_test_data(&[V1_RUST, EMPTY_UNCOMPRESSED].join("/"))
+    input_file_for_test_data(&[RUST_TESTDATA, EMPTY_UNCOMPRESSED].join("/"))
 }
 
 pub(crate) fn java_uncompressed_metric_input_file() -> InputFile {
-    input_file_for_test_data(&[V1_JAVA, METRIC_UNCOMPRESSED].join("/"))
+    input_file_for_test_data(&[JAVA_TESTDATA, METRIC_UNCOMPRESSED].join("/"))
 }
 
 pub(crate) fn rust_uncompressed_metric_input_file() -> InputFile {
-    input_file_for_test_data(&[V1_RUST, METRIC_UNCOMPRESSED].join("/"))
+    input_file_for_test_data(&[RUST_TESTDATA, METRIC_UNCOMPRESSED].join("/"))
 }
 
 pub(crate) fn java_zstd_compressed_metric_input_file() -> InputFile {
-    input_file_for_test_data(&[V1_JAVA, METRIC_ZSTD_COMPRESSED].join("/"))
+    input_file_for_test_data(&[JAVA_TESTDATA, METRIC_ZSTD_COMPRESSED].join("/"))
 }
 
 pub(crate) fn rust_zstd_compressed_metric_input_file() -> InputFile {
-    input_file_for_test_data(&[V1_RUST, METRIC_ZSTD_COMPRESSED].join("/"))
+    input_file_for_test_data(&[RUST_TESTDATA, METRIC_ZSTD_COMPRESSED].join("/"))
 }
 
 pub(crate) fn empty_footer_payload() -> FileMetadata {

crates/puffin/src/writer.rs → crates/iceberg/src/puffin/writer.rs

@@ -20,13 +20,13 @@ use std::sync::atomic::AtomicU64;
 use std::sync::Arc;
 
 use bytes::Bytes;
-use iceberg::io::{FileWrite, OutputFile};
-use iceberg::writer::file_writer::TrackWriter;
-use iceberg::{Error, ErrorKind, Result};
 
-use crate::blob::Blob;
-use crate::compression::CompressionCodec;
-use crate::metadata::{BlobMetadata, ByteNumber, FileMetadata, Flag};
+use crate::io::{FileWrite, OutputFile};
+use crate::puffin::blob::Blob;
+use crate::puffin::compression::CompressionCodec;
+use crate::puffin::metadata::{BlobMetadata, ByteNumber, FileMetadata, Flag};
+use crate::writer::file_writer::track_writer::TrackWriter;
+use crate::{Error, ErrorKind, Result};
 
 /// Puffin writer
 pub struct PuffinWriter {
@@ -186,21 +186,21 @@ impl PuffinWriter {
 mod tests {
     use std::collections::HashMap;
 
-    use iceberg::io::{FileIOBuilder, InputFile, OutputFile};
-    use iceberg::Result;
     use tempfile::TempDir;
 
-    use crate::blob::Blob;
-    use crate::compression::CompressionCodec;
-    use crate::metadata::FileMetadata;
-    use crate::test_utils::{
+    use crate::io::{FileIOBuilder, InputFile, OutputFile};
+    use crate::puffin::blob::Blob;
+    use crate::puffin::compression::CompressionCodec;
+    use crate::puffin::metadata::FileMetadata;
+    use crate::puffin::test_utils::{
         blob_0, blob_1, empty_footer_payload, empty_footer_payload_bytes, file_properties,
         java_empty_uncompressed_input_file, java_uncompressed_metric_input_file,
         java_zstd_compressed_metric_input_file, uncompressed_metric_file_metadata,
         zstd_compressed_metric_file_metadata,
     };
-    use crate::writer::PuffinWriter;
-    use crate::PuffinReader;
+    use crate::puffin::writer::PuffinWriter;
+    use crate::puffin::PuffinReader;
+    use crate::Result;
 
     #[tokio::test]
     async fn test_throws_error_if_attempt_to_add_blob_after_closing() {

crates/iceberg/src/writer/file_writer/mod.rs

@@ -27,8 +27,7 @@ use crate::Result;
 mod parquet_writer;
 pub use parquet_writer::{ParquetWriter, ParquetWriterBuilder};
 
-mod track_writer;
-pub use track_writer::TrackWriter;
+pub(crate) mod track_writer;
 
 pub mod location_generator;
 

crates/iceberg/src/writer/file_writer/track_writer.rs

@@ -24,7 +24,7 @@ use crate::io::FileWrite;
 use crate::Result;
 
 /// `TrackWriter` is used to track the written size.
-pub struct TrackWriter {
+pub(crate) struct TrackWriter {
     inner: Box<dyn FileWrite>,
     written_size: Arc<AtomicU64>,
 }