Merge branch 'dev-rebase-main-20241119' of https://github.com/risingw…

…avelabs/iceberg-rust into li0k/test_iceberg_catalog2

.asf.yaml

@@ -48,9 +48,8 @@ github:
     issues: true
     projects: true
   collaborators:
-    - Xuanwo
-    - liurenjie1024
     - JanKaul
+    - c-thiel
   ghp_branch: gh-pages
   ghp_path: /
 

.github/workflows/ci_typos.yml

@@ -42,4 +42,4 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check typos
-        uses: crate-ci/typos@v1.24.6
+        uses: crate-ci/typos@v1.27.3

Cargo.toml

@@ -18,11 +18,12 @@
 [workspace]
 resolver = "2"
 members = [
-  "crates/catalog/*",
-  "crates/examples",
-  "crates/iceberg",
-  "crates/integrations/*",
-  "crates/test_utils",
+    "crates/catalog/*",
+    "crates/e2e_test",
+    "crates/examples",
+    "crates/iceberg",
+    "crates/integrations/*",
+    "crates/test_utils",
 ]
 exclude = ["bindings/python"]
 
@@ -41,6 +42,7 @@ apache-avro = "0.17"
 array-init = "2"
 arrow-arith = { version = "53" }
 arrow-array = { version = "53" }
+arrow-cast = { version = "53" }
 arrow-ord = { version = "53" }
 arrow-schema = { version = "53" }
 arrow-select = { version = "53" }
@@ -69,10 +71,11 @@ itertools = "0.13"
 log = "0.4"
 mockito = "1"
 murmur3 = "0.5.2"
+num-bigint = "0.4.6"
 once_cell = "1"
-opendal = "0.50"
+opendal = "0.50.1"
 ordered-float = "4"
-parquet = "53"
+parquet = "53.1"
 paste = "1"
 pilota = "0.11.2"
 pretty_assertions = "1.4"

NOTICE

@@ -1,5 +1,5 @@
 Apache Iceberg Rust
-Copyright 2023 The Apache Software Foundation
+Copyright 2023-2024 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

README.md

@@ -81,13 +81,13 @@ The Apache Iceberg Rust project is composed of the following components:
 Iceberg Rust is built and tested with stable rust, and will keep a rolling MSRV(minimum supported rust version). The 
 current MSRV is 1.77.1. 
 
-Also, we use unstable rust to run linters, such as `clippy` and `rustfmt`. But this will not affect downstream users, 
+Also, iceberg-rust use unstable rust to run linters, such as `clippy` and `rustfmt`. But this will not affect downstream users, 
 and only MSRV is required.  
 
 
 ## Contribute
 
-Apache Iceberg is an active open-source project, governed under the Apache Software Foundation (ASF). We are always open to people who want to use or contribute to it. Here are some ways to get involved.
+Apache Iceberg is an active open-source project, governed under the Apache Software Foundation (ASF). Iceberg-rust are always open to people who want to use or contribute to it. Here are some ways to get involved.
 
 - Start with [Contributing Guide](CONTRIBUTING.md).
 - Submit [Issues](https://github.com/apache/iceberg-rust/issues/new) for bug report or feature requests.

crates/catalog/glue/tests/glue_catalog_test.rs

@@ -237,7 +237,7 @@ async fn test_create_table() -> Result<()> {
     assert!(
         catalog
             .file_io()
-            .is_exist("s3a://warehouse/hive/metadata/")
+            .exists("s3a://warehouse/hive/metadata/")
             .await?
     );
 

crates/catalog/hms/tests/hms_catalog_test.rs

@@ -212,7 +212,7 @@ async fn test_create_table() -> Result<()> {
     assert!(
         catalog
             .file_io()
-            .is_exist("s3a://warehouse/hive/metadata/")
+            .exists("s3a://warehouse/hive/metadata/")
             .await?
     );
 

crates/catalog/memory/src/catalog.rs

@@ -283,7 +283,7 @@ mod tests {
     use std::iter::FromIterator;
 
     use iceberg::io::FileIOBuilder;
-    use iceberg::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, Type};
+    use iceberg::spec::{BoundPartitionSpec, NestedField, PrimitiveType, Schema, SortOrder, Type};
     use regex::Regex;
     use tempfile::TempDir;
 
@@ -355,7 +355,7 @@ mod tests {
 
         assert_eq!(metadata.current_schema().as_ref(), expected_schema);
 
-        let expected_partition_spec = PartitionSpec::builder(expected_schema)
+        let expected_partition_spec = BoundPartitionSpec::builder((*expected_schema).clone())
             .with_spec_id(0)
             .build()
             .unwrap();
@@ -365,7 +365,7 @@ mod tests {
                 .partition_specs_iter()
                 .map(|p| p.as_ref())
                 .collect_vec(),
-            vec![&expected_partition_spec]
+            vec![&expected_partition_spec.into_schemaless()]
         );
 
         let expected_sorted_order = SortOrder::builder()

crates/catalog/rest/src/catalog.rs

@@ -1376,7 +1376,7 @@ mod tests {
             .with_schema_id(0)
             .with_summary(Summary {
                 operation: Operation::Append,
-                other: HashMap::from_iter([
+                additional_properties: HashMap::from_iter([
                     ("spark.app.id", "local-1646787004168"),
                     ("added-data-files", "1"),
                     ("added-records", "1"),

crates/catalog/sql/Cargo.toml

@@ -32,15 +32,15 @@ keywords = ["iceberg", "sql", "catalog"]
 async-trait = { workspace = true }
 iceberg = { workspace = true }
 serde_json = { workspace = true }
-sqlx = { version = "0.7.4", features = ["any"], default-features = false }
+sqlx = { version = "0.8.1", features = ["any"], default-features = false }
 typed-builder = { workspace = true }
 uuid = { workspace = true, features = ["v4"] }
 
 [dev-dependencies]
 iceberg_test_utils = { path = "../../test_utils", features = ["tests"] }
 itertools = { workspace = true }
 regex = "1.10.5"
-sqlx = { version = "0.7.4", features = [
+sqlx = { version = "0.8.1", features = [
   "tls-rustls",
   "runtime-tokio",
   "any",