feat: add source trait (#803)

Adds the `source` trait for sources to implement. The idea being they'll be created as async functions and we can just call next on the returned `stream` -- no need to make them a whole `ReadPipeline` or anything we think. * Adds the `sparrow-sources` crate * Adds the `sparrow-interfaces` crate * Refactors `batch` into `sparrow-batch`. --------- Co-authored-by: Ben Chambers <[email protected]>
kaskada-ai · Oct 12, 2023 · 13be46f · 13be46f
1 parent 0445cb0
commit 13be46f
Show file tree

Hide file tree

Showing 36 changed files with 812 additions and 293 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/sparrow-arrow/src/lib.rs b/crates/sparrow-arrow/src/lib.rs
@@ -10,18 +10,12 @@
 pub mod attachments;
 #[cfg(feature = "avro")]
 pub mod avro;
-mod batch;
 mod concat_take;
 pub mod downcast;
 pub mod hash;
 pub mod hasher;
-mod row_time;
 pub mod scalar_value;
 pub mod serde;
-#[cfg(any(test, feature = "testing"))]
-pub mod testing;
 pub mod utils;
 
-pub use batch::*;
 pub use concat_take::*;
-pub use row_time::*;
diff --git a/crates/sparrow-batch/Cargo.toml b/crates/sparrow-batch/Cargo.toml
@@ -0,0 +1,30 @@
+[package]
+name = "sparrow-batch"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish = false
+description = """
+Defines the main struct for wrapping RecordBatches in execution.
+"""
+
+[dependencies]
+arrow.workspace = true
+arrow-array.workspace = true
+arrow-schema.workspace = true
+arrow-select.workspace = true
+derive_more.workspace = true
+error-stack.workspace = true
+itertools.workspace = true
+static_init.workspace = true
+proptest = { workspace = true, optional = true }
+
+[dev-dependencies]
+sparrow-testing = { path = "../sparrow-testing" }
+sparrow-merge = { path = "../sparrow-merge" }
+proptest.workspace = true
+
+[lib]
+bench = false
+doctest = false