implement aws sdk credential provider

datafusion_iceberg/examples/dataframe.rs

@@ -3,9 +3,9 @@ use datafusion::arrow::record_batch::RecordBatch;
 use datafusion::dataframe::DataFrameWriteOptions;
 use datafusion::prelude::SessionContext;
 use datafusion_iceberg::catalog::catalog::IcebergCatalog;
-use iceberg_rust::catalog::bucket::ObjectStoreBuilder;
 use iceberg_rust::catalog::identifier::Identifier;
 use iceberg_rust::catalog::Catalog;
+use iceberg_rust::object_store::ObjectStoreBuilder;
 use iceberg_rust::spec::schema::Schema;
 use iceberg_rust::spec::types::{StructField, StructType};
 use iceberg_rust::table::Table;

datafusion_iceberg/examples/insert_csv.rs

@@ -6,7 +6,7 @@ use datafusion::{
     execution::{context::SessionContext, SessionStateBuilder},
 };
 use datafusion_expr::ScalarUDF;
-use iceberg_rust::catalog::bucket::ObjectStoreBuilder;
+use iceberg_rust::object_store::ObjectStoreBuilder;
 use iceberg_sql_catalog::SqlCatalogList;
 
 use datafusion_iceberg::{

datafusion_iceberg/examples/insert_table.rs

@@ -1,7 +1,8 @@
 use datafusion::{arrow::array::Int64Array, prelude::SessionContext};
 use datafusion_iceberg::DataFusionTable;
 use iceberg_rust::{
-    catalog::{bucket::ObjectStoreBuilder, Catalog},
+    catalog::Catalog,
+    object_store::ObjectStoreBuilder,
     spec::{
         partition::{PartitionField, PartitionSpec, Transform},
         schema::Schema,

datafusion_iceberg/examples/refresh_materialized_view.rs

@@ -1,9 +1,9 @@
 use datafusion::{arrow::array::Int64Array, prelude::SessionContext};
 use datafusion_iceberg::catalog::catalog::IcebergCatalog;
 use datafusion_iceberg::materialized_view::refresh_materialized_view;
-use iceberg_rust::catalog::bucket::ObjectStoreBuilder;
 use iceberg_rust::catalog::CatalogList;
 use iceberg_rust::materialized_view::MaterializedView;
+use iceberg_rust::object_store::ObjectStoreBuilder;
 use iceberg_rust::spec::partition::PartitionSpec;
 use iceberg_rust::spec::view_metadata::{Version, ViewRepresentation};
 use iceberg_rust::spec::{

datafusion_iceberg/src/catalog/mirror.rs

@@ -7,14 +7,14 @@ use std::{collections::HashSet, sync::Arc};
 use iceberg_rust::spec::{tabular::TabularMetadata, view_metadata::REF_PREFIX};
 use iceberg_rust::{
     catalog::{
-        bucket::Bucket,
         create::{CreateMaterializedView, CreateView},
         identifier::Identifier,
         namespace::Namespace,
         tabular::Tabular,
         Catalog,
     },
     error::Error as IcebergError,
+    object_store::Bucket,
     spec::table_metadata::new_metadata_location,
 };
 

datafusion_iceberg/src/materialized_view.rs

@@ -217,14 +217,6 @@ pub async fn refresh_materialized_view(
 mod tests {
 
     use datafusion::{arrow::array::Int64Array, prelude::SessionContext};
-    use iceberg_rust::{
-        catalog::bucket::ObjectStoreBuilder,
-        spec::{
-            partition::{PartitionField, Transform},
-            schema::Schema,
-            types::{PrimitiveType, StructField, StructType, Type},
-        },
-    };
     use iceberg_rust::{
         catalog::CatalogList,
         materialized_view::MaterializedView,
@@ -234,6 +226,14 @@ mod tests {
         },
         table::Table,
     };
+    use iceberg_rust::{
+        object_store::ObjectStoreBuilder,
+        spec::{
+            partition::{PartitionField, Transform},
+            schema::Schema,
+            types::{PrimitiveType, StructField, StructType, Type},
+        },
+    };
     use iceberg_sql_catalog::SqlCatalogList;
     use std::sync::Arc;
 

datafusion_iceberg/src/planner.rs

@@ -640,7 +640,7 @@ mod tests {
         execution::{context::SessionContext, SessionStateBuilder},
     };
     use datafusion_expr::ScalarUDF;
-    use iceberg_rust::catalog::bucket::ObjectStoreBuilder;
+    use iceberg_rust::object_store::ObjectStoreBuilder;
     use iceberg_sql_catalog::SqlCatalogList;
     use tokio::time::sleep;
 

datafusion_iceberg/src/table.rs

@@ -590,6 +590,15 @@ impl DataSink for IcebergDataSink {
 mod tests {
 
     use datafusion::{arrow::array::Int64Array, prelude::SessionContext};
+    use iceberg_rust::{
+        catalog::tabular::Tabular,
+        object_store::ObjectStoreBuilder,
+        spec::{
+            partition::{PartitionField, Transform},
+            schema::Schema,
+            types::{PrimitiveType, StructField, StructType, Type},
+        },
+    };
     use iceberg_rust::{
         catalog::Catalog,
         spec::{
@@ -599,14 +608,6 @@ mod tests {
         table::Table,
         view::View,
     };
-    use iceberg_rust::{
-        catalog::{bucket::ObjectStoreBuilder, tabular::Tabular},
-        spec::{
-            partition::{PartitionField, Transform},
-            schema::Schema,
-            types::{PrimitiveType, StructField, StructType, Type},
-        },
-    };
     use iceberg_sql_catalog::SqlCatalog;
 
     use std::{ops::Deref, sync::Arc};

datafusion_iceberg/tests/integration_trino.rs

@@ -9,9 +9,9 @@ use datafusion::execution::context::SessionContext;
 use datafusion_iceberg::catalog::catalog::IcebergCatalog;
 use iceberg_rest_catalog::apis::configuration::Configuration;
 use iceberg_rest_catalog::catalog::RestCatalog;
-use iceberg_rust::catalog::bucket::ObjectStoreBuilder;
 use iceberg_rust::catalog::namespace::Namespace;
 use iceberg_rust::catalog::Catalog;
+use iceberg_rust::object_store::ObjectStoreBuilder;
 use iceberg_rust::spec::partition::{PartitionField, PartitionSpec, Transform};
 use iceberg_rust::spec::schema::Schema;
 use iceberg_rust::spec::types::{PrimitiveType, StructField, StructType, Type};
@@ -183,7 +183,7 @@ async fn integration_trino_rest() {
         .await
         .unwrap();
 
-    let object_store = ObjectStoreBuilder::aws()
+    let object_store = ObjectStoreBuilder::s3()
         .with_config("aws_access_key_id".parse().unwrap(), "user")
         .with_config("aws_secret_access_key".parse().unwrap(), "password")
         .with_config(

iceberg-file-catalog/src/lib.rs

@@ -8,7 +8,6 @@ use async_trait::async_trait;
 use futures::{future, TryStreamExt};
 use iceberg_rust::{
     catalog::{
-        bucket::{Bucket, ObjectStoreBuilder},
         commit::{
             apply_table_updates, apply_view_updates, check_table_requirements,
             check_view_requirements, CommitTable, CommitView, TableRequirement,
@@ -21,6 +20,7 @@ use iceberg_rust::{
     },
     error::Error as IcebergError,
     materialized_view::MaterializedView,
+    object_store::{Bucket, ObjectStoreBuilder},
     spec::{
         materialized_view_metadata::MaterializedViewMetadata,
         table_metadata::{new_metadata_location, TableMetadata},
@@ -628,7 +628,10 @@ pub mod tests {
         catalog::catalog::IcebergCatalog,
         planner::{iceberg_transform, IcebergQueryPlanner},
     };
-    use iceberg_rust::catalog::{bucket::ObjectStoreBuilder, namespace::Namespace, Catalog};
+    use iceberg_rust::{
+        catalog::{namespace::Namespace, Catalog},
+        object_store::ObjectStoreBuilder,
+    };
     use std::sync::Arc;
     // use testcontainers::{core::ExecCommand, runners::AsyncRunner, ImageExt};
     // use testcontainers_modules::localstack::LocalStack;

iceberg-glue-catalog/src/lib.rs

@@ -11,7 +11,6 @@ use aws_sdk_glue::{
 };
 use iceberg_rust::{
     catalog::{
-        bucket::{Bucket, ObjectStoreBuilder},
         commit::{
             apply_table_updates, apply_view_updates, check_table_requirements,
             check_view_requirements, CommitTable, CommitView, TableRequirement,
@@ -24,6 +23,7 @@ use iceberg_rust::{
     },
     error::Error as IcebergError,
     materialized_view::MaterializedView,
+    object_store::{Bucket, ObjectStoreBuilder},
     spec::{
         self,
         materialized_view_metadata::MaterializedViewMetadata,
@@ -910,7 +910,10 @@ pub mod tests {
         catalog::catalog::IcebergCatalog,
         planner::{iceberg_transform, IcebergQueryPlanner},
     };
-    use iceberg_rust::catalog::{bucket::ObjectStoreBuilder, namespace::Namespace, Catalog};
+    use iceberg_rust::{
+        catalog::{namespace::Namespace, Catalog},
+        object_store::ObjectStoreBuilder,
+    };
 
     use std::sync::Arc;
     use testcontainers::{

iceberg-rest-catalog/src/catalog.rs

@@ -5,7 +5,6 @@ Iceberg rest catalog implementation
 */
 use iceberg_rust::{
     catalog::{
-        bucket::{Bucket, ObjectStoreBuilder},
         commit::CommitView,
         create::{CreateMaterializedView, CreateTable, CreateView},
         identifier::{self, Identifier},
@@ -15,6 +14,7 @@ use iceberg_rust::{
     },
     error::Error,
     materialized_view::MaterializedView,
+    object_store::{Bucket, ObjectStoreBuilder},
     spec::{
         materialized_view_metadata::MaterializedViewMetadata,
         table_metadata::TableMetadata,
@@ -516,9 +516,8 @@ impl CatalogList for RestCatalogList {
 #[cfg(test)]
 pub mod tests {
     use iceberg_rust::{
-        catalog::{
-            bucket::ObjectStoreBuilder, identifier::Identifier, namespace::Namespace, Catalog,
-        },
+        catalog::{identifier::Identifier, namespace::Namespace, Catalog},
+        object_store::ObjectStoreBuilder,
         spec::{
             schema::Schema,
             types::{PrimitiveType, StructField, StructType, Type},

iceberg-rust/Cargo.toml

@@ -31,6 +31,8 @@ thiserror = { workspace = true }
 derive-getters = { workspace = true }
 iceberg-rust-spec = { path = "../iceberg-rust-spec", version = "0.5.8" }
 smallvec = { version = "1.13.2", features = ["const_generics"] }
+aws-config = "1.5.10"
+aws-credential-types = "1.2.1"
 
 [dev-dependencies]
 chrono = { workspace = true }

iceberg-rust/src/arrow/write.rs

@@ -29,7 +29,7 @@ use parquet::{
 };
 use uuid::Uuid;
 
-use crate::{catalog::bucket::Bucket, error::Error, file_format::parquet::parquet_to_datafile};
+use crate::{error::Error, file_format::parquet::parquet_to_datafile, object_store::Bucket};
 
 use super::partition::partition_record_batches;
 

iceberg-rust/src/catalog/mod.rs

@@ -14,13 +14,12 @@ use crate::materialized_view::MaterializedView;
 use crate::table::Table;
 use crate::view::View;
 
-use self::bucket::Bucket;
 use self::commit::{CommitTable, CommitView};
 use self::create::{CreateMaterializedView, CreateTable, CreateView};
 use self::namespace::Namespace;
 use self::tabular::Tabular;
+use crate::object_store::Bucket;
 
-pub mod bucket;
 pub mod commit;
 pub mod create;
 pub mod tabular;

iceberg-rust/src/lib.rs

@@ -7,6 +7,7 @@ pub mod catalog;
 pub mod error;
 pub mod file_format;
 pub mod materialized_view;
+pub mod object_store;
 pub mod spec;
 pub mod sql;
 pub mod store;

iceberg-rust/src/materialized_view/mod.rs

@@ -11,10 +11,10 @@ use object_store::ObjectStore;
 
 use crate::{
     catalog::{
-        bucket::Bucket, create::CreateMaterializedViewBuilder, identifier::Identifier,
-        tabular::Tabular, Catalog,
+        create::CreateMaterializedViewBuilder, identifier::Identifier, tabular::Tabular, Catalog,
     },
     error::Error,
+    object_store::Bucket,
 };
 
 use self::{storage_table::StorageTable, transaction::Transaction as MaterializedViewTransaction};