feat: implement manifest and manifest list writer adapter (cont'd) (#216)

Author: dongxiao1198

src/iceberg/CMakeLists.txt

@@ -18,6 +18,7 @@
 set(ICEBERG_INCLUDES "$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/src>"
                      "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>")
 set(ICEBERG_SOURCES
+    arrow_c_data_guard_internal.cc
     catalog/memory/in_memory_catalog.cc
     expression/expression.cc
     expression/expressions.cc
@@ -28,8 +29,12 @@ set(ICEBERG_SOURCES
     file_writer.cc
     inheritable_metadata.cc
     json_internal.cc
+    manifest_adapter.cc
     manifest_entry.cc
     manifest_list.cc
+    manifest_reader.cc
+    manifest_reader_internal.cc
+    manifest_writer.cc
     metadata_columns.cc
     name_mapping.cc
     partition_field.cc
@@ -51,16 +56,15 @@ set(ICEBERG_SOURCES
     transform.cc
     transform_function.cc
     type.cc
-    manifest_reader.cc
-    manifest_reader_internal.cc
-    manifest_writer.cc
-    arrow_c_data_guard_internal.cc
     util/conversions.cc
     util/decimal.cc
     util/gzip_internal.cc
     util/murmurhash3_internal.cc
     util/timepoint.cc
-    util/uuid.cc)
+    util/uuid.cc
+    v1_metadata.cc
+    v2_metadata.cc
+    v3_metadata.cc)
 
 set(ICEBERG_STATIC_BUILD_INTERFACE_LIBS)
 set(ICEBERG_SHARED_BUILD_INTERFACE_LIBS)

src/iceberg/arrow/arrow_fs_file_io.cc

@@ -22,9 +22,9 @@
 #include <arrow/filesystem/localfs.h>
 #include <arrow/filesystem/mockfs.h>
 
-#include "iceberg/arrow/arrow_error_transform_internal.h"
 #include "iceberg/arrow/arrow_file_io.h"
 #include "iceberg/arrow/arrow_fs_file_io_internal.h"
+#include "iceberg/arrow/arrow_status_internal.h"
 
 namespace iceberg::arrow {
 

src/iceberg/arrow/arrow_error_transform_internal.h renamed to src/iceberg/arrow/arrow_status_internal.h

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#define ICEBERG_NANOARROW_RETURN_UNEXPECTED(status)                  \
+  if (status != NANOARROW_OK) [[unlikely]] {                         \
+    return iceberg::InvalidArrowData("nanoarrow error: {}", status); \
+  }
+
+#define ICEBERG_NANOARROW_RETURN_UNEXPECTED_WITH_ERROR(status, error)       \
+  if (status != NANOARROW_OK) [[unlikely]] {                                \
+    return iceberg::InvalidArrowData("nanoarrow error: {} msg: {}", status, \
+                                     error.message);                        \
+  }

src/iceberg/arrow/nanoarrow_status_internal.h

@@ -32,7 +32,7 @@
 #include <avro/NodeImpl.hh>
 #include <avro/Types.hh>
 
-#include "iceberg/arrow/arrow_error_transform_internal.h"
+#include "iceberg/arrow/arrow_status_internal.h"
 #include "iceberg/avro/avro_data_util_internal.h"
 #include "iceberg/avro/avro_schema_util_internal.h"
 #include "iceberg/schema.h"

src/iceberg/avro/avro_data_util.cc

@@ -31,8 +31,8 @@
 #include <avro/Generic.hh>
 #include <avro/GenericDatum.hh>
 
-#include "iceberg/arrow/arrow_error_transform_internal.h"
 #include "iceberg/arrow/arrow_fs_file_io_internal.h"
+#include "iceberg/arrow/arrow_status_internal.h"
 #include "iceberg/avro/avro_data_util_internal.h"
 #include "iceberg/avro/avro_register.h"
 #include "iceberg/avro/avro_schema_util_internal.h"

src/iceberg/avro/avro_reader.cc

@@ -29,8 +29,8 @@
 #include <avro/Generic.hh>
 #include <avro/GenericDatum.hh>
 
-#include "iceberg/arrow/arrow_error_transform_internal.h"
 #include "iceberg/arrow/arrow_fs_file_io_internal.h"
+#include "iceberg/arrow/arrow_status_internal.h"
 #include "iceberg/avro/avro_data_util_internal.h"
 #include "iceberg/avro/avro_register.h"
 #include "iceberg/avro/avro_schema_util_internal.h"
@@ -75,9 +75,9 @@ class AvroWriter::Impl {
     return {};
   }
 
-  Status Write(ArrowArray data) {
+  Status Write(ArrowArray* data) {
     ICEBERG_ARROW_ASSIGN_OR_RETURN(auto result,
-                                   ::arrow::ImportArray(&data, &arrow_schema_));
+                                   ::arrow::ImportArray(data, &arrow_schema_));
 
     for (int64_t i = 0; i < result->length(); i++) {
       ICEBERG_RETURN_UNEXPECTED(ExtractDatumFromArray(*result, i, datum_.get()));
@@ -119,7 +119,7 @@ class AvroWriter::Impl {
 
 AvroWriter::~AvroWriter() = default;
 
-Status AvroWriter::Write(ArrowArray data) { return impl_->Write(data); }
+Status AvroWriter::Write(ArrowArray* data) { return impl_->Write(data); }
 
 Status AvroWriter::Open(const WriterOptions& options) {
   impl_ = std::make_unique<Impl>();

src/iceberg/avro/avro_writer.cc

@@ -35,7 +35,7 @@ class ICEBERG_BUNDLE_EXPORT AvroWriter : public Writer {
 
   Status Close() final;
 
-  Status Write(ArrowArray data) final;
+  Status Write(ArrowArray* data) final;
 
   std::optional<Metrics> metrics() final;
 

src/iceberg/avro/avro_writer.h

@@ -65,7 +65,8 @@ class ICEBERG_EXPORT Writer {
   /// \brief Write arrow data to the file.
   ///
   /// \return Status of write results.
-  virtual Status Write(ArrowArray data) = 0;
+  /// \note Ownership of the data is transferred to the writer.
+  virtual Status Write(ArrowArray* data) = 0;
 
   /// \brief Get the file statistics.
   /// Only valid after the file is closed.