Relocating test_utils and test data into the deltalake-test crate

This will ensure that alll of this testdata and utility code can be used
for cloud specific integration testing

crates/deltalake-aws/Cargo.toml

@@ -24,9 +24,11 @@ url = { workspace = true }
 
 [dev-dependencies]
 serial_test = "2"
+deltalake-test = { path = "../deltalake-test" }
 
 [features]
 default = ["rustls"]
+integration_test = []
 native-tls = [
     "rusoto_core/native-tls",
     "rusoto_sts/native-tls",

crates/deltalake-aws/tests/integration_read.rs

@@ -0,0 +1,13 @@
+
+
+#[tokio::test]
+#[serial]
+async fn test_read_tables_aws() -> TestResult {
+    read_tables(StorageIntegration::Amazon).await?;
+
+    for (prefix, prefix_encoded) in TEST_PREFIXES.iter().zip(TEST_PREFIXES_ENCODED.iter()) {
+        read_table_paths(StorageIntegration::Amazon, prefix, prefix_encoded).await?;
+    }
+
+    Ok(())
+}

crates/deltalake-core/tests/repair_s3_rename_test.rs → crates/deltalake-aws/tests/repair_s3_rename_test.rs

@@ -1,7 +1,4 @@
-#![cfg(all(
-    any(feature = "s3", feature = "s3-native-tls"),
-    feature = "integration_test"
-))]
+#![cfg(feature = "integration_test")]
 use bytes::Bytes;
 use deltalake_core::test_utils::{IntegrationContext, StorageIntegration};
 use deltalake_core::{storage::s3::S3StorageBackend, DeltaTableBuilder, ObjectStore};

crates/deltalake-core/Cargo.toml

@@ -117,8 +117,11 @@ fs_extra = { version = "1.3.0", optional = true }
 tempdir = { version = "0", optional = true }
 
 [dev-dependencies]
+criterion = "0.5"
 ctor = "0"
+deltalake-test = { path = "../deltalake-test" }
 dotenvy = "0"
+hyper = { version = "0.14", features = ["server"] }
 maplit = "1"
 pretty_assertions = "1.2.1"
 rand = "0.8"
@@ -127,8 +130,6 @@ tempdir = "0"
 tempfile = "3"
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
 utime = "0.3"
-hyper = { version = "0.14", features = ["server"] }
-criterion = "0.5"
 
 [features]
 azure = ["object_store/azure"]

crates/deltalake-core/src/lib.rs

@@ -128,11 +128,6 @@ pub use parquet2;
 #[cfg(all(feature = "arrow", feature = "parquet"))]
 pub use protocol::checkpoints;
 
-// needed only for integration tests
-// TODO can / should we move this into the test crate?
-#[cfg(feature = "integration_test")]
-pub mod test_utils;
-
 /// Creates and loads a DeltaTable from the given path with current metadata.
 /// Infers the storage backend to use from the scheme in the given table path.
 ///

crates/deltalake-core/src/storage/mod.rs

@@ -24,6 +24,7 @@ pub use datafusion::execution::object_store::DefaultObjectStoreRegistry;
 #[cfg(feature = "datafusion")]
 pub(crate) use datafusion::execution::object_store::ObjectStoreRegistry;
 
+pub use object_store;
 pub use object_store::path::{Path, DELIMITER};
 pub use object_store::{
     DynObjectStore, Error as ObjectStoreError, GetResult, ListResult, MultipartId, ObjectMeta,

crates/deltalake-core/tests/command_vacuum.rs

@@ -1,15 +1,13 @@
 use chrono::Duration;
-use common::clock::TestClock;
-use common::TestContext;
 use deltalake_core::kernel::StructType;
 use deltalake_core::operations::vacuum::Clock;
 use deltalake_core::operations::DeltaOps;
+use deltalake_test::clock::TestClock;
+use deltalake_test::*;
 use object_store::{path::Path, Error as ObjectStoreError, ObjectStore};
 use serde_json::json;
 use std::sync::Arc;
 
-mod common;
-
 /// Basic schema
 pub fn get_xy_date_schema() -> StructType {
     serde_json::from_value(json!({
@@ -51,7 +49,7 @@ async fn test_non_partitioned_table() {
     ];
 
     for path in paths {
-        common::add_file(
+        add_file(
             &mut table,
             &path,
             "random junk".as_bytes().into(),
@@ -64,7 +62,7 @@ async fn test_non_partitioned_table() {
 
     clock.tick(Duration::seconds(10));
 
-    common::remove_file(
+    remove_file(
         &mut table,
         "delete_me.parquet",
         &[],
@@ -103,7 +101,7 @@ async fn test_partitioned_table() {
     let partition_values = [("date", Some("2022-07-03")), ("x", Some("2"))];
 
     for path in paths {
-        common::add_file(
+        add_file(
             &mut table,
             &path,
             "random junk".as_bytes().into(),
@@ -116,7 +114,7 @@ async fn test_partitioned_table() {
 
     clock.tick(Duration::seconds(10));
 
-    common::remove_file(
+    remove_file(
         &mut table,
         "date=2022-07-03/x=2/delete_me.parquet",
         &partition_values,
@@ -168,7 +166,7 @@ async fn test_partitions_included() {
     let partition_values = &[("_date", Some("2022-07-03"))];
 
     for path in paths {
-        common::add_file(
+        add_file(
             &mut table,
             &path,
             "random junk".as_bytes().into(),
@@ -181,7 +179,7 @@ async fn test_partitions_included() {
 
     clock.tick(Duration::seconds(10));
 
-    common::remove_file(
+    remove_file(
         &mut table,
         "_date=2022-07-03/delete_me.parquet",
         partition_values,
@@ -247,7 +245,7 @@ async fn test_non_managed_files() {
     ];
 
     for path in paths_delete.iter().chain(paths_ignore.iter()) {
-        common::add_file(
+        add_file(
             &mut table,
             path,
             "random junk".as_bytes().into(),

crates/deltalake-core/tests/integration_object_store.rs

@@ -2,8 +2,8 @@
 
 use bytes::Bytes;
 use deltalake_core::storage::utils::flatten_list_stream;
-use deltalake_core::test_utils::{IntegrationContext, StorageIntegration, TestResult};
 use deltalake_core::{DeltaTableBuilder, ObjectStore};
+use deltalake_test::utils::{IntegrationContext, StorageIntegration, TestResult};
 use object_store::{path::Path, DynObjectStore, Error as ObjectStoreError};
 use serial_test::serial;
 

crates/deltalake-core/tests/integration_read.rs

@@ -1,9 +1,7 @@
 #![cfg(feature = "integration_test")]
 
-use deltalake_core::test_utils::{IntegrationContext, StorageIntegration, TestResult, TestTables};
 use deltalake_core::{DeltaTableBuilder, ObjectStore};
-#[cfg(any(feature = "s3", feature = "s3-native-tls"))]
-#[cfg(any(feature = "s3", feature = "s3-native-tls"))]
+use deltalake_test::utils::{IntegrationContext, StorageIntegration, TestResult, TestTables};
 use maplit::hashmap;
 use object_store::path::Path;
 use serial_test::serial;
@@ -102,22 +100,6 @@ mod hdfs {
     }
 }
 
-#[cfg(any(feature = "s3", feature = "s3-native-tls"))]
-mod s3 {
-    use super::*;
-    #[tokio::test]
-    #[serial]
-    async fn test_read_tables_aws() -> TestResult {
-        read_tables(StorageIntegration::Amazon).await?;
-
-        for (prefix, prefix_encoded) in TEST_PREFIXES.iter().zip(TEST_PREFIXES_ENCODED.iter()) {
-            read_table_paths(StorageIntegration::Amazon, prefix, prefix_encoded).await?;
-        }
-
-        Ok(())
-    }
-}
-
 async fn read_tables(storage: StorageIntegration) -> TestResult {
     let context = IntegrationContext::new(storage)?;
     context.load_table(TestTables::Simple).await?;

crates/deltalake-test/.gitignore

@@ -0,0 +1,12 @@
+target/
+/.idea/
+*.bat
+tests/data/checkpoints_tombstones/expired/
+tests/data/checkpoints_tombstones/metadata_broken/
+tests/data/checkpoints_tombstones/metadata_false/
+tests/data/checkpoints_tombstones/metadata_true/
+tests/data/checkpoints_with_expired_logs/
+tests/data/read_null_partitions_from_checkpoint/
+tests/data/action_reconciliation/
+tests/data/simple_table_with_no_checkpoint/
+tests/data/simple_table_with_no_checkpoint_2/

crates/deltalake-test/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "deltalake-test"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+bytes = { workspace = true }
+chrono = { workspace = true, default-features = false, features = ["clock"] }
+deltalake-core = { path = "../deltalake-core" }
+dotenvy = "0"
+fs_extra = "1.3.0"
+rand = "0.8"
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+tempdir = "0"
+tempfile = "3"
+
+[features]
+default = []
+datafusion = [
+        "deltalake-core/datafusion",
+]

crates/deltalake-core/tests/common/datafusion.rs → crates/deltalake-test/src/datafusion.rs

@@ -1,6 +1,6 @@
-use datafusion::execution::context::{SessionContext, SessionState};
-use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv};
-use datafusion::prelude::SessionConfig;
+use deltalake_core::datafusion::execution::context::{SessionContext, SessionState};
+use deltalake_core::datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv};
+use deltalake_core::datafusion::prelude::SessionConfig;
 use deltalake_core::delta_datafusion::DeltaTableFactory;
 use std::sync::Arc;
 

crates/deltalake-core/tests/common/mod.rs → crates/deltalake-test/src/lib.rs

@@ -8,21 +8,16 @@ use deltalake_core::operations::transaction::commit;
 use deltalake_core::protocol::{DeltaOperation, SaveMode};
 use deltalake_core::DeltaTable;
 use deltalake_core::DeltaTableBuilder;
-use object_store::{path::Path, ObjectStore};
+use deltalake_core::{ObjectStore, Path};
 use std::any::Any;
 use std::collections::HashMap;
 use std::sync::Arc;
 use tempdir::TempDir;
 
-#[cfg(feature = "azure")]
-pub mod adls;
 pub mod clock;
 #[cfg(feature = "datafusion")]
 pub mod datafusion;
-#[cfg(feature = "hdfs")]
-pub mod hdfs;
-#[cfg(any(feature = "s3", feature = "s3-native-tls"))]
-pub mod s3;
+pub mod utils;
 
 #[derive(Default)]
 pub struct TestContext {
@@ -47,8 +42,6 @@ impl TestContext {
             Ok("LOCALFS") | Err(std::env::VarError::NotPresent) => setup_local_context().await,
             #[cfg(feature = "azure")]
             Ok("AZURE_GEN2") => adls::setup_azure_gen2_context().await,
-            #[cfg(any(feature = "s3", feature = "s3-native-tls"))]
-            Ok("S3_LOCAL_STACK") => s3::setup_s3_context().await,
             #[cfg(feature = "hdfs")]
             Ok("HDFS") => hdfs::setup_hdfs_context(),
             _ => panic!("Invalid backend for delta-rs tests"),