Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update to DataFusion 42, Arrow 53.1 (RC), remove DataFrameDataSource Python logic #513

Merged
merged 19 commits into from
Oct 6, 2024
Merged
Prev Previous commit
Next Next commit
more test fixes
jonmmease committed Oct 5, 2024
commit 4a6d51528aeb16fd2120b22dec48c301434694bd
2 changes: 1 addition & 1 deletion vegafusion-python/src/lib.rs
Original file line number Diff line number Diff line change
@@ -566,7 +566,7 @@ impl PyVegaFusionRuntime {

let warnings = pythonize::pythonize(py, &warnings)?;

Ok((tx_spec.into(), datasets.into(), warnings.into()))
Ok((tx_spec.into(), datasets, warnings.into()))
})
}

18 changes: 9 additions & 9 deletions vegafusion-sql/tests/test_fold.rs
Original file line number Diff line number Diff line change
@@ -43,16 +43,16 @@ mod test_simple_fold {
let df_result = if let Ok(df) = df_result {
df.sort(
vec![
Expr::Sort(expr::Sort {
expr: Box::new(flat_col("country")),
expr::Sort {
expr: flat_col("country"),
asc: true,
nulls_first: true,
}),
Expr::Sort(expr::Sort {
expr: Box::new(flat_col("key")),
},
expr::Sort {
expr: flat_col("key"),
asc: true,
nulls_first: true,
}),
},
],
None,
)
@@ -100,11 +100,11 @@ mod test_ordered_fold {

let df_result = if let Ok(df) = df_result {
df.sort(
vec![Expr::Sort(expr::Sort {
expr: Box::new(flat_col("_order")),
vec![expr::Sort {
expr: flat_col("_order"),
asc: true,
nulls_first: true,
})],
}],
None,
)
.await
2 changes: 1 addition & 1 deletion vegafusion-sql/tests/test_joinaggregate.rs
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
extern crate lazy_static;

mod utils;
use datafusion_expr::{expr, Expr};
use datafusion_expr::expr;
use rstest::rstest;
use rstest_reuse::{self, *};
use serde_json::json;
16 changes: 8 additions & 8 deletions vegafusion-sql/tests/test_select.rs
Original file line number Diff line number Diff line change
@@ -84,7 +84,7 @@ mod test_numeric_operators {
#[cfg(test)]
mod test_logical_operators {
use crate::*;
use datafusion_expr::{expr, lit, Expr};
use datafusion_expr::{expr, lit};
use vegafusion_common::column::flat_col;

#[apply(dialect_names)]
@@ -211,7 +211,7 @@ mod test_between {
mod test_cast_numeric {
use crate::*;
use arrow::datatypes::DataType;
use datafusion_expr::{cast, expr, Expr};
use datafusion_expr::{cast, expr};
use vegafusion_common::column::flat_col;

#[apply(dialect_names)]
@@ -266,7 +266,7 @@ mod test_cast_numeric {
mod test_try_cast_numeric {
use crate::*;
use arrow::datatypes::DataType;
use datafusion_expr::{expr, try_cast, Expr};
use datafusion_expr::{expr, try_cast};
use vegafusion_common::column::flat_col;

#[apply(dialect_names)]
@@ -327,7 +327,7 @@ mod test_try_cast_numeric {
mod test_cast_string {
use crate::*;
use arrow::datatypes::DataType;
use datafusion_expr::{cast, expr, Expr};
use datafusion_expr::{cast, expr};
use vegafusion_common::column::flat_col;

#[apply(dialect_names)]
@@ -377,7 +377,7 @@ mod test_cast_string {
mod test_try_cast_string {
use crate::*;
use arrow::datatypes::DataType;
use datafusion_expr::{expr, try_cast, Expr};
use datafusion_expr::{expr, try_cast};
use vegafusion_common::column::flat_col;

#[apply(dialect_names)]
@@ -432,7 +432,7 @@ mod test_try_cast_string {
#[cfg(test)]
mod test_non_finite_numbers {
use crate::*;
use datafusion_expr::{expr, lit, Expr};
use datafusion_expr::{expr, lit};
use vegafusion_common::column::flat_col;

#[apply(dialect_names)]
@@ -485,7 +485,7 @@ mod test_non_finite_numbers {
#[cfg(test)]
mod test_scalar_math_functions {
use crate::*;
use datafusion_expr::{expr, Expr};
use datafusion_expr::expr;
use datafusion_functions::math::expr_fn::{abs, acos, asin, tan};
use vegafusion_common::column::flat_col;

@@ -1433,7 +1433,7 @@ mod test_timestamp_to_utc_timestamp {
#[cfg(test)]
mod test_string_ops {
use crate::*;
use datafusion_expr::{expr, lit, Expr};
use datafusion_expr::{expr, lit};
use datafusion_functions::string::expr_fn::{concat, lower, upper};
use vegafusion_common::column::flat_col;

68 changes: 37 additions & 31 deletions vegafusion-sql/tests/test_window.rs
Original file line number Diff line number Diff line change
@@ -40,11 +40,11 @@ mod test_simple_aggs_unbounded {
.unwrap();

let df = SqlDataFrame::from_values(&table, conn, Default::default()).unwrap();
let order_by = vec![Expr::Sort(expr::Sort {
expr: Box::new(flat_col("a")),
let order_by = vec![expr::Sort {
expr: flat_col("a"),
asc: true,
nulls_first: true,
})];
}];
let window_frame = WindowFrame::new(Some(true));
let df_result = df
.select(vec![
@@ -144,11 +144,11 @@ mod test_simple_aggs_unbounded_groups {
.unwrap();

let df = SqlDataFrame::from_values(&table, conn, Default::default()).unwrap();
let order_by = vec![Expr::Sort(expr::Sort {
expr: Box::new(flat_col("a")),
let order_by = vec![expr::Sort {
expr: flat_col("a"),
asc: true,
nulls_first: true,
})];
}];
let window_frame = WindowFrame::new_bounds(
WindowFrameUnits::Groups,
WindowFrameBound::Preceding(ScalarValue::Null),
@@ -253,11 +253,11 @@ mod test_simple_aggs_bounded {
.unwrap();

let df = SqlDataFrame::from_values(&table, conn, Default::default()).unwrap();
let order_by = vec![Expr::Sort(expr::Sort {
expr: Box::new(flat_col("a")),
let order_by = vec![expr::Sort {
expr: flat_col("a"),
asc: true,
nulls_first: true,
})];
}];
let window_frame = WindowFrame::new_bounds(
WindowFrameUnits::Rows,
WindowFrameBound::Preceding(ScalarValue::from(1)),
@@ -361,11 +361,11 @@ mod test_simple_aggs_bounded_groups {
.unwrap();

let df = SqlDataFrame::from_values(&table, conn, Default::default()).unwrap();
let order_by = vec![Expr::Sort(expr::Sort {
expr: Box::new(col("a")),
let order_by = vec![expr::Sort {
expr: col("a"),
asc: true,
nulls_first: true,
})];
}];
let window_frame = WindowFrame::new_bounds(
WindowFrameUnits::Groups,
WindowFrameBound::Preceding(ScalarValue::from(1)),
@@ -427,16 +427,16 @@ mod test_simple_aggs_bounded_groups {
let df_result = if let Ok(df) = df_result {
df.sort(
vec![
Expr::Sort(expr::Sort {
expr: Box::new(col("a")),
expr::Sort {
expr: col("a"),
asc: true,
nulls_first: true,
}),
Expr::Sort(expr::Sort {
expr: Box::new(col("b")),
},
expr::Sort {
expr: col("b"),
asc: true,
nulls_first: true,
}),
},
],
None,
)
@@ -465,6 +465,9 @@ mod test_simple_window_fns {

#[apply(dialect_names)]
async fn test(dialect_name: &str) {
use std::sync::Arc;

use datafusion::functions_window::row_number::RowNumber;
use sqlparser::ast::NullTreatment;

println!("{dialect_name}");
@@ -480,20 +483,20 @@ mod test_simple_window_fns {
.unwrap();

let df = SqlDataFrame::from_values(&table, conn, Default::default()).unwrap();
let order_by = vec![Expr::Sort(expr::Sort {
expr: Box::new(col("a")),
let order_by = vec![expr::Sort {
expr: col("a"),
asc: true,
nulls_first: true,
})];
}];
let window_frame = WindowFrame::new(Some(true));
let df_result = df
.select(vec![
col("a"),
col("b"),
col("c"),
Expr::WindowFunction(expr::WindowFunction {
fun: WindowFunctionDefinition::BuiltInWindowFunction(
BuiltInWindowFunction::RowNumber,
fun: WindowFunctionDefinition::WindowUDF(
Arc::new(RowNumber::new().into()),
),
args: vec![],
partition_by: vec![],
@@ -590,11 +593,11 @@ mod test_advanced_window_fns {
.unwrap();

let df = SqlDataFrame::from_values(&table, conn, Default::default()).unwrap();
let order_by = vec![Expr::Sort(expr::Sort {
expr: Box::new(col("a")),
let order_by = vec![expr::Sort {
expr: col("a"),
asc: true,
nulls_first: true,
})];
}];
let window_frame = WindowFrame::new(Some(true));
let df_result = df
.select(vec![
@@ -685,6 +688,9 @@ mod test_unordered_row_number {

#[apply(dialect_names)]
async fn test(dialect_name: &str) {
use std::sync::Arc;

use datafusion::functions_window::row_number::RowNumber;
use sqlparser::ast::NullTreatment;

println!("{dialect_name}");
@@ -700,20 +706,20 @@ mod test_unordered_row_number {
.unwrap();

let df = SqlDataFrame::from_values(&table, conn, Default::default()).unwrap();
let order_by = vec![Expr::Sort(expr::Sort {
expr: Box::new(col("a")),
let order_by = vec![expr::Sort {
expr: col("a"),
asc: true,
nulls_first: true,
})];
}];
let window_frame = WindowFrame::new(Some(true));
let df_result = df
.select(vec![
col("a"),
col("b"),
col("c"),
Expr::WindowFunction(expr::WindowFunction {
fun: WindowFunctionDefinition::BuiltInWindowFunction(
BuiltInWindowFunction::RowNumber,
fun: WindowFunctionDefinition::WindowUDF(
Arc::new(RowNumber::new().into()),
),
args: vec![],
partition_by: vec![],