Skip to content

Commit

Permalink
[FEAT] connect: remove excessive warnings from spark connect (#3499)
Browse files Browse the repository at this point in the history
it seems that the `common` is **always** set, producing the warning for
every single operation when using spark connect. This changes it so that
it'll only warn if `common.origin` is set.
  • Loading branch information
universalmind303 authored Dec 5, 2024
1 parent 33baba8 commit 9739bb6
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 4 deletions.
4 changes: 3 additions & 1 deletion src/daft-connect/src/translation/expr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ mod unresolved_function;

pub fn to_daft_expr(expression: &Expression) -> eyre::Result<daft_dsl::ExprRef> {
if let Some(common) = &expression.common {
warn!("Ignoring common metadata for relation: {common:?}; not yet implemented");
if common.origin.is_some() {
warn!("Ignoring common metadata for relation: {common:?}; not yet implemented");
}
};

let Some(expr) = &expression.expr_type else {
Expand Down
4 changes: 3 additions & 1 deletion src/daft-connect/src/translation/logical_plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ impl From<LogicalPlanBuilder> for Plan {

pub fn to_logical_plan(relation: Relation) -> eyre::Result<Plan> {
if let Some(common) = relation.common {
warn!("Ignoring common metadata for relation: {common:?}; not yet implemented");
if common.origin.is_some() {
warn!("Ignoring common metadata for relation: {common:?}; not yet implemented");
}
};

let Some(rel_type) = relation.rel_type else {
Expand Down
6 changes: 4 additions & 2 deletions src/daft-connect/src/translation/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ use crate::translation::{to_logical_plan, to_spark_datatype};

#[tracing::instrument(skip_all)]
pub fn relation_to_schema(input: Relation) -> eyre::Result<DataType> {
if input.common.is_some() {
warn!("We do not currently look at common fields");
if let Some(common) = &input.common {
if common.origin.is_some() {
warn!("Ignoring common metadata for relation: {common:?}; not yet implemented");
}
}

let plan = to_logical_plan(input)?;
Expand Down

0 comments on commit 9739bb6

Please sign in to comment.