Skip to content

Commit

Permalink
fix: use arrow's schema instead of spark's for local rel (#3602)
Browse files Browse the repository at this point in the history
Co-authored-by: Andrew Gazelka <[email protected]>
  • Loading branch information
universalmind303 and andrewgazelka authored Dec 19, 2024
1 parent ca4d3f7 commit e0d2b8a
Show file tree
Hide file tree
Showing 7 changed files with 72 additions and 426 deletions.
45 changes: 0 additions & 45 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 0 additions & 2 deletions src/daft-connect/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
[dependencies]
arrow2 = {workspace = true, features = ["io_json_integration"]}
async-stream = "0.3.6"
color-eyre = "0.6.3"
common-daft-config = {workspace = true}
common-file-formats = {workspace = true}
daft-core = {workspace = true}
Expand All @@ -18,7 +17,6 @@ eyre = "0.6.12"
futures = "0.3.31"
itertools = {workspace = true}
pyo3 = {workspace = true, optional = true}
serde_json = {workspace = true}
spark-connect = {workspace = true}
tokio = {version = "1.40.0", features = ["full"]}
tonic = "0.12.3"
Expand Down
2 changes: 1 addition & 1 deletion src/daft-connect/src/translation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ mod literal;
mod logical_plan;
mod schema;

pub use datatype::{deser_spark_datatype, to_daft_datatype, to_spark_datatype};
pub use datatype::{to_daft_datatype, to_spark_datatype};
pub use expr::to_daft_expr;
pub use literal::to_daft_literal;
pub use logical_plan::SparkAnalyzer;
Expand Down
3 changes: 0 additions & 3 deletions src/daft-connect/src/translation/datatype.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@ use eyre::{bail, ensure, WrapErr};
use spark_connect::data_type::Kind;
use tracing::warn;

mod codec;
pub use codec::deser as deser_spark_datatype;

pub fn to_spark_datatype(datatype: &DataType) -> spark_connect::DataType {
match datatype {
DataType::Null => spark_connect::DataType {
Expand Down
241 changes: 0 additions & 241 deletions src/daft-connect/src/translation/datatype/codec.rs

This file was deleted.

Loading

0 comments on commit e0d2b8a

Please sign in to comment.