From 30ae6090ade0f2d54ea79ff35b392592d8a8915c Mon Sep 17 00:00:00 2001 From: Ian Joiner <14581281+iajoiner@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:11:41 -0400 Subject: [PATCH] feat: make polars postprocessing optional behind feature flag --- .github/workflows/lint-and-test.yml | 8 ++ Cargo.toml | 4 +- crates/proof-of-sql/Cargo.toml | 6 +- crates/proof-of-sql/src/base/database/mod.rs | 2 + .../database/record_batch_test_accessor.rs | 9 +- .../src/sql/parse/result_expr_builder.rs | 7 +- .../src/sql/transform/group_by_expr.rs | 48 ++++++---- crates/proof-of-sql/src/sql/transform/mod.rs | 11 +++ .../src/sql/transform/order_by_exprs.rs | 15 ++- .../src/sql/transform/record_batch_expr.rs | 7 +- .../src/sql/transform/result_expr.rs | 10 +- .../src/sql/transform/select_expr.rs | 92 ++++++++++++++----- .../src/sql/transform/slice_expr.rs | 7 ++ .../src/sql/transform/test_utility.rs | 11 +-- 14 files changed, 172 insertions(+), 65 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 628a8f59d..4d729345c 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -48,6 +48,14 @@ jobs: run: cargo check -p proof-of-sql --no-default-features --features="test" - name: Run cargo check (proof-of-sql) (just "blitzar" feature) run: cargo check -p proof-of-sql --no-default-features --features="blitzar" + - name: Run cargo check (proof-of-sql) (just "polars" feature) + run: cargo check -p proof-of-sql --no-default-features --features="polars" + - name: Run cargo check (proof-of-sql) (no "test" feature) + run: cargo check -p proof-of-sql --no-default-features --features="blitzar polars" + - name: Run cargo check (proof-of-sql) (no "blitzar" feature) + run: cargo check -p proof-of-sql --no-default-features --features="test polars" + - name: Run cargo check (proof-of-sql) (no "polars" feature) + run: cargo check -p proof-of-sql --no-default-features --features="blitzar test" test: name: Test Suite diff --git a/Cargo.toml b/Cargo.toml index 1913112a2..aa6f52a11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ bigdecimal = { version = "0.4.5", features = ["serde"] } blake3 = { version = "1.3.3" } blitzar = { version = "3.0.2" } bumpalo = { version = "3.11.0" } -bytemuck = {version = "1.14.2" } +bytemuck = {version = "1.14.2", features = ["derive"] } byte-slice-cast = { version = "1.2.1" } clap = { version = "4.5.4" } criterion = { version = "0.5.1" } @@ -35,7 +35,7 @@ derive_more = { version = "0.99" } dyn_partial_eq = { version = "0.1.2" } flexbuffers = { version = "2.0.0" } hashbrown = { version = "0.14.0" } -indexmap = { version = "2.1" } +indexmap = { version = "2.1", features = ["serde"] } itertools = { version = "0.13.0" } lalrpop-util = { version = "0.20.0" } lazy_static = { version = "1.4.0" } diff --git a/crates/proof-of-sql/Cargo.toml b/crates/proof-of-sql/Cargo.toml index d6a0736e9..c8157bf28 100644 --- a/crates/proof-of-sql/Cargo.toml +++ b/crates/proof-of-sql/Cargo.toml @@ -40,7 +40,7 @@ lazy_static = { workspace = true } merlin = { workspace = true } num-traits = { workspace = true } num-bigint = { workspace = true, default-features = false } -polars = { workspace = true, features = ["lazy", "bigidx", "dtype-decimal", "serde-lazy"] } +polars = { workspace = true, features = ["lazy", "bigidx", "dtype-decimal", "serde-lazy"], optional = true } postcard = { workspace = true, features = ["alloc"] } proof-of-sql-parser = { workspace = true } rand = { workspace = true, optional = true } @@ -59,7 +59,7 @@ clap = { workspace = true, features = ["derive"] } criterion = { workspace = true, features = ["html_reports"] } opentelemetry = { workspace = true } opentelemetry-jaeger = { workspace = true } -polars = { workspace = true, features = ["lazy"] } +polars = { workspace = true, features = ["lazy", "dtype-decimal"] } rand = { workspace = true } rand_core = { workspace = true } serde_json = { workspace = true } @@ -69,7 +69,7 @@ tracing-subscriber = { workspace = true } flexbuffers = { workspace = true } [features] -default = ["blitzar"] +default = ["blitzar", "polars"] test = ["dep:rand"] [lints] diff --git a/crates/proof-of-sql/src/base/database/mod.rs b/crates/proof-of-sql/src/base/database/mod.rs index 7ecd72c5a..3d76a63e1 100644 --- a/crates/proof-of-sql/src/base/database/mod.rs +++ b/crates/proof-of-sql/src/base/database/mod.rs @@ -16,7 +16,9 @@ pub use table_ref::TableRef; mod arrow_array_to_column_conversion; pub use arrow_array_to_column_conversion::{ArrayRefExt, ArrowArrayToColumnConversionError}; +#[cfg(any(test, feature = "polars"))] mod record_batch_dataframe_conversion; +#[cfg(any(test, feature = "polars"))] pub(crate) use record_batch_dataframe_conversion::{ dataframe_to_record_batch, record_batch_to_dataframe, }; diff --git a/crates/proof-of-sql/src/base/database/record_batch_test_accessor.rs b/crates/proof-of-sql/src/base/database/record_batch_test_accessor.rs index a67766ae4..cb58d2501 100644 --- a/crates/proof-of-sql/src/base/database/record_batch_test_accessor.rs +++ b/crates/proof-of-sql/src/base/database/record_batch_test_accessor.rs @@ -1,13 +1,15 @@ +#[cfg(any(test, feature = "polars"))] +use super::{dataframe_to_record_batch, record_batch_to_dataframe}; use super::{ - dataframe_to_record_batch, record_batch_to_dataframe, ArrayRefExt, Column, ColumnRef, - ColumnType, CommitmentAccessor, DataAccessor, MetadataAccessor, SchemaAccessor, TableRef, - TestAccessor, + ArrayRefExt, Column, ColumnRef, ColumnType, CommitmentAccessor, DataAccessor, MetadataAccessor, + SchemaAccessor, TableRef, TestAccessor, }; use crate::base::scalar::{compute_commitment_for_testing, Curve25519Scalar}; use arrow::{array::ArrayRef, datatypes::DataType, record_batch::RecordBatch}; use bumpalo::Bump; use curve25519_dalek::ristretto::RistrettoPoint; use indexmap::IndexMap; +#[cfg(any(test, feature = "polars"))] use polars::prelude::DataFrame; use proof_of_sql_parser::Identifier; use std::collections::HashMap; @@ -114,6 +116,7 @@ impl TestAccessor for RecordBatchTestAccessor { impl RecordBatchTestAccessor { /// Apply a query function to table and then convert the result to a RecordBatch + #[cfg(any(test, feature = "polars"))] pub fn query_table( &self, table_ref: TableRef, diff --git a/crates/proof-of-sql/src/sql/parse/result_expr_builder.rs b/crates/proof-of-sql/src/sql/parse/result_expr_builder.rs index 0dda9bb26..89b4755f9 100644 --- a/crates/proof-of-sql/src/sql/parse/result_expr_builder.rs +++ b/crates/proof-of-sql/src/sql/parse/result_expr_builder.rs @@ -36,13 +36,10 @@ impl ResultExprBuilder { .iter() .map(|aliased_expr| Expression::Column(aliased_expr.alias)) .collect(); - self.composition - .add(Box::new(SelectExpr::new_from_expressions(&exprs))); + self.composition.add(Box::new(SelectExpr::new(&exprs))); } else { self.composition - .add(Box::new(SelectExpr::new_from_aliased_result_exprs( - aliased_exprs, - ))); + .add(Box::new(SelectExpr::new(aliased_exprs))); } self } diff --git a/crates/proof-of-sql/src/sql/transform/group_by_expr.rs b/crates/proof-of-sql/src/sql/transform/group_by_expr.rs index 56cc7d3e7..87a541f70 100644 --- a/crates/proof-of-sql/src/sql/transform/group_by_expr.rs +++ b/crates/proof-of-sql/src/sql/transform/group_by_expr.rs @@ -1,7 +1,10 @@ #[allow(deprecated)] +#[cfg(feature = "polars")] use super::DataFrameExpr; +#[cfg(feature = "polars")] use super::{ToPolarsExpr, INT128_PRECISION, INT128_SCALE}; use dyn_partial_eq::DynPartialEq; +#[cfg(feature = "polars")] use polars::prelude::{col, DataType, Expr, GetOutput, LazyFrame, NamedFrom, Series}; use proof_of_sql_parser::{intermediate_ast::AliasedResultExpr, Identifier}; use serde::{Deserialize, Serialize}; @@ -10,32 +13,43 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, DynPartialEq, PartialEq, Serialize, Deserialize)] pub struct GroupByExpr { /// A list of aggregation column expressions - agg_exprs: Vec, + aliased_exprs: Vec, /// A list of group by column expressions - by_exprs: Vec, + by_ids: Vec, } impl GroupByExpr { /// Create a new group by expression containing the group by and aggregation expressions pub fn new(by_ids: &[Identifier], aliased_exprs: &[AliasedResultExpr]) -> Self { - let by_exprs = Vec::from_iter(by_ids.iter().map(|id| col(id.as_str()))); - let agg_exprs = Vec::from_iter(aliased_exprs.iter().map(ToPolarsExpr::to_polars_expr)); - assert!(!agg_exprs.is_empty(), "Agg expressions must not be empty"); assert!( - !by_exprs.is_empty(), - "Group by expressions must not be empty" + !aliased_exprs.is_empty(), + "Agg expressions must not be empty" ); + assert!(!by_ids.is_empty(), "Group by expressions must not be empty"); Self { - by_exprs, - agg_exprs, + by_ids: by_ids.to_vec(), + aliased_exprs: aliased_exprs.to_vec(), } } + + #[cfg(feature = "polars")] + fn agg_exprs(&self) -> Vec { + self.aliased_exprs + .iter() + .map(ToPolarsExpr::to_polars_expr) + .collect() + } } +#[cfg(not(feature = "polars"))] +#[typetag::serde] +impl super::RecordBatchExpr for GroupByExpr {} +#[cfg(feature = "polars")] super::impl_record_batch_expr_for_data_frame_expr!(GroupByExpr); #[allow(deprecated)] +#[cfg(feature = "polars")] impl DataFrameExpr for GroupByExpr { fn lazy_transformation(&self, lazy_frame: LazyFrame, num_input_rows: usize) -> LazyFrame { // TODO: polars currently lacks support for min/max aggregation in data frames @@ -43,23 +57,23 @@ impl DataFrameExpr for GroupByExpr { // We remove the group by clause to temporarily work around this limitation. // Issue created to track progress: https://github.com/pola-rs/polars/issues/11232 if num_input_rows == 0 { - return lazy_frame.select(&self.agg_exprs).limit(0); + return lazy_frame.select(&self.agg_exprs()).limit(0); } if num_input_rows == 1 { - return lazy_frame.select(&self.agg_exprs); + return lazy_frame.select(&self.agg_exprs()); } // Add invalid column aliases to group by expressions so that we can // exclude them from the final result. - let by_expr_aliases = (0..self.by_exprs.len()) + let by_expr_aliases = (0..self.by_ids.len()) .map(|pos| "#$".to_owned() + pos.to_string().as_str()) .collect::>(); let by_exprs: Vec<_> = self - .by_exprs - .clone() - .into_iter() + .by_ids + .iter() + .map(|id| col(id.as_str())) .zip(by_expr_aliases.iter()) .map(|(expr, alias)| expr.alias(alias.as_str())) // TODO: remove this mapping once Polars supports decimal columns inside group by @@ -71,11 +85,12 @@ impl DataFrameExpr for GroupByExpr { // to avoid non-deterministic results with our tests. lazy_frame .group_by_stable(&by_exprs) - .agg(&self.agg_exprs) + .agg(&self.agg_exprs()) .select(&[col("*").exclude(by_expr_aliases)]) } } +#[cfg(any(test, feature = "polars"))] pub(crate) fn group_by_map_i128_to_utf8(v: i128) -> String { // use big end to allow // skipping leading zeros @@ -99,6 +114,7 @@ pub(crate) fn group_by_map_i128_to_utf8(v: i128) -> String { // Polars doesn't support Decimal columns inside group by. // So we need to remap them to the supported UTF8 type. +#[cfg(feature = "polars")] fn group_by_map_to_utf8_if_decimal(expr: Expr) -> Expr { expr.map( |series| match series.dtype().clone() { diff --git a/crates/proof-of-sql/src/sql/transform/mod.rs b/crates/proof-of-sql/src/sql/transform/mod.rs index 6b832da8b..99b3236d4 100644 --- a/crates/proof-of-sql/src/sql/transform/mod.rs +++ b/crates/proof-of-sql/src/sql/transform/mod.rs @@ -1,8 +1,10 @@ //! This module contains postprocessing for non-provable components. /// The precision for [ColumnType::INT128] values +#[cfg(feature = "polars")] pub const INT128_PRECISION: usize = 38; /// The scale for [ColumnType::INT128] values +#[cfg(feature = "polars")] pub const INT128_SCALE: usize = 0; mod result_expr; @@ -17,10 +19,13 @@ pub use composition_expr::CompositionExpr; #[cfg(test)] pub mod composition_expr_test; +#[cfg(feature = "polars")] mod data_frame_expr; #[allow(deprecated)] +#[cfg(feature = "polars")] pub(crate) use data_frame_expr::DataFrameExpr; mod record_batch_expr; +#[cfg(feature = "polars")] pub(crate) use record_batch_expr::impl_record_batch_expr_for_data_frame_expr; pub use record_batch_expr::RecordBatchExpr; @@ -53,10 +58,16 @@ pub use group_by_expr::GroupByExpr; #[cfg(test)] mod group_by_expr_test; +#[cfg(feature = "polars")] mod polars_conversions; +#[cfg(feature = "polars")] pub use polars_conversions::LiteralConversion; +#[cfg(feature = "polars")] mod polars_arithmetic; +#[cfg(feature = "polars")] pub use polars_arithmetic::SafeDivision; +#[cfg(feature = "polars")] mod to_polars_expr; +#[cfg(feature = "polars")] pub(crate) use to_polars_expr::ToPolarsExpr; diff --git a/crates/proof-of-sql/src/sql/transform/order_by_exprs.rs b/crates/proof-of-sql/src/sql/transform/order_by_exprs.rs index 8c2132ec8..3ad484d8a 100644 --- a/crates/proof-of-sql/src/sql/transform/order_by_exprs.rs +++ b/crates/proof-of-sql/src/sql/transform/order_by_exprs.rs @@ -1,10 +1,16 @@ #[allow(deprecated)] +#[cfg(feature = "polars")] use super::DataFrameExpr; +#[cfg(feature = "polars")] use super::{INT128_PRECISION, INT128_SCALE}; +#[cfg(any(test, feature = "polars"))] use arrow::datatypes::ArrowNativeTypeOp; use dyn_partial_eq::DynPartialEq; +#[cfg(feature = "polars")] use polars::prelude::{col, DataType, Expr, GetOutput, LazyFrame, NamedFrom, Series}; -use proof_of_sql_parser::intermediate_ast::{OrderBy, OrderByDirection}; +use proof_of_sql_parser::intermediate_ast::OrderBy; +#[cfg(feature = "polars")] +use proof_of_sql_parser::intermediate_ast::OrderByDirection; use serde::{Deserialize, Serialize}; /// A node representing a list of `OrderBy` expressions. @@ -20,8 +26,13 @@ impl OrderByExprs { } } +#[cfg(not(feature = "polars"))] +#[typetag::serde] +impl super::RecordBatchExpr for OrderByExprs {} +#[cfg(feature = "polars")] super::impl_record_batch_expr_for_data_frame_expr!(OrderByExprs); #[allow(deprecated)] +#[cfg(feature = "polars")] impl DataFrameExpr for OrderByExprs { /// Sort the `LazyFrame` by the `OrderBy` expressions. fn lazy_transformation(&self, lazy_frame: LazyFrame, _: usize) -> LazyFrame { @@ -51,6 +62,7 @@ impl DataFrameExpr for OrderByExprs { /// * `a < b` if and only if `map_i128_to_utf8(a) < map_i128_to_utf8(b)`. /// * `a == b` if and only if `map_i128_to_utf8(a) == map_i128_to_utf8(b)`. /// * `a > b` if and only if `map_i128_to_utf8(a) > map_i128_to_utf8(b)`. +#[cfg(any(test, feature = "polars"))] pub(crate) fn order_by_map_i128_to_utf8(v: i128) -> String { let is_neg = v.is_negative() as u8; v.abs() @@ -78,6 +90,7 @@ pub(crate) fn order_by_map_i128_to_utf8(v: i128) -> String { // Polars doesn't support Decimal columns inside order by. // So we need to remap them to the supported UTF8 type. +#[cfg(feature = "polars")] fn order_by_map_to_utf8_if_decimal(expr: Expr) -> Expr { expr.map( |series| match series.dtype().clone() { diff --git a/crates/proof-of-sql/src/sql/transform/record_batch_expr.rs b/crates/proof-of-sql/src/sql/transform/record_batch_expr.rs index 0e59e2aea..fbdc9ff7d 100644 --- a/crates/proof-of-sql/src/sql/transform/record_batch_expr.rs +++ b/crates/proof-of-sql/src/sql/transform/record_batch_expr.rs @@ -7,9 +7,13 @@ use std::fmt::Debug; #[dyn_partial_eq] pub trait RecordBatchExpr: Debug + Send + Sync { /// Apply the transformation to the `RecordBatch` and return the result. - fn apply_transformation(&self, record_batch: RecordBatch) -> Option; + #[allow(unused_variables)] + fn apply_transformation(&self, record_batch: RecordBatch) -> Option { + None + } } +#[cfg(feature = "polars")] macro_rules! impl_record_batch_expr_for_data_frame_expr { ($t:ty) => { #[typetag::serde] @@ -29,4 +33,5 @@ macro_rules! impl_record_batch_expr_for_data_frame_expr { }; } +#[cfg(feature = "polars")] pub(crate) use impl_record_batch_expr_for_data_frame_expr; diff --git a/crates/proof-of-sql/src/sql/transform/result_expr.rs b/crates/proof-of-sql/src/sql/transform/result_expr.rs index 244a0afec..5c8d38481 100644 --- a/crates/proof-of-sql/src/sql/transform/result_expr.rs +++ b/crates/proof-of-sql/src/sql/transform/result_expr.rs @@ -1,9 +1,9 @@ -use crate::{ - base::database::{dataframe_to_record_batch, record_batch_to_dataframe}, - sql::transform::RecordBatchExpr, -}; +#[cfg(feature = "polars")] +use crate::base::database::{dataframe_to_record_batch, record_batch_to_dataframe}; +use crate::sql::transform::RecordBatchExpr; use arrow::record_batch::RecordBatch; use dyn_partial_eq::DynPartialEq; +#[cfg(feature = "polars")] use polars::prelude::{IntoLazy, LazyFrame}; use serde::{Deserialize, Serialize}; @@ -23,11 +23,13 @@ impl ResultExpr { } } +#[cfg(feature = "polars")] pub(super) fn record_batch_to_lazy_frame(result_batch: RecordBatch) -> Option<(LazyFrame, usize)> { let num_input_rows = result_batch.num_rows(); let df = record_batch_to_dataframe(result_batch)?; Some((df.lazy(), num_input_rows)) } +#[cfg(feature = "polars")] pub(super) fn lazy_frame_to_record_batch(lazy_frame: LazyFrame) -> Option { // We're currently excluding NULLs in post-processing due to a lack of // prover support, aiming to avoid future complexities. diff --git a/crates/proof-of-sql/src/sql/transform/select_expr.rs b/crates/proof-of-sql/src/sql/transform/select_expr.rs index 3c407fd40..a2300c560 100644 --- a/crates/proof-of-sql/src/sql/transform/select_expr.rs +++ b/crates/proof-of-sql/src/sql/transform/select_expr.rs @@ -1,48 +1,86 @@ #[allow(deprecated)] +#[cfg(feature = "polars")] use super::DataFrameExpr; +use super::RecordBatchExpr; +#[cfg(feature = "polars")] use super::{ - record_batch_expr::RecordBatchExpr, result_expr::{lazy_frame_to_record_batch, record_batch_to_lazy_frame}, ToPolarsExpr, }; use arrow::record_batch::RecordBatch; use dyn_partial_eq::DynPartialEq; +#[cfg(feature = "polars")] use polars::prelude::{Expr, LazyFrame}; use proof_of_sql_parser::intermediate_ast::{AliasedResultExpr, Expression}; use serde::{Deserialize, Serialize}; +#[derive(Debug, DynPartialEq, PartialEq, Serialize, Deserialize)] +pub enum SelectTerm { + #[cfg(test)] + #[cfg(feature = "polars")] + Polars(Expr), + AliasedResult(AliasedResultExpr), + Result(Expression), +} +#[cfg(feature = "polars")] +impl ToPolarsExpr for SelectTerm { + fn to_polars_expr(&self) -> Expr { + match self { + #[cfg(test)] + Self::Polars(s) => s.to_polars_expr(), + Self::AliasedResult(s) => s.to_polars_expr(), + Self::Result(s) => s.to_polars_expr(), + } + } +} +#[cfg(test)] +#[cfg(feature = "polars")] +impl From<&Expr> for SelectTerm { + fn from(value: &Expr) -> Self { + Self::Polars(value.clone()) + } +} +impl From<&AliasedResultExpr> for SelectTerm { + fn from(value: &AliasedResultExpr) -> Self { + Self::AliasedResult(value.clone()) + } +} +impl From<&Expression> for SelectTerm { + fn from(value: &Expression) -> Self { + Self::Result(value.clone()) + } +} +#[cfg(test)] +impl From<&Box> for SelectTerm { + fn from(value: &Box) -> Self { + value.as_ref().into() + } +} + /// The select expression used to select, reorder, and apply alias transformations #[derive(Debug, DynPartialEq, PartialEq, Serialize, Deserialize)] pub struct SelectExpr { /// The schema of the resulting lazy frame - result_schema: Vec, + result_schema: Vec, } impl SelectExpr { - #[cfg(test)] - pub(crate) fn new(exprs: &[impl ToPolarsExpr]) -> Self { - Self::new_from_to_polars(exprs) - } - fn new_from_to_polars(exprs: &[impl ToPolarsExpr]) -> Self { - let result_schema = Vec::from_iter(exprs.iter().map(ToPolarsExpr::to_polars_expr)); + /// Create a new select expression from a slice that implements `Into` + pub fn new(exprs: impl IntoIterator>) -> Self { + let result_schema = Vec::from_iter(exprs.into_iter().map(Into::into)); assert!(!result_schema.is_empty()); Self { result_schema } } - /// Create a new select expression from a slice of AliasedResultExpr - pub fn new_from_aliased_result_exprs(aliased_exprs: &[AliasedResultExpr]) -> Self { - Self::new_from_to_polars(aliased_exprs) - } - /// Create a new select expression from a slice of Expressions - pub fn new_from_expressions(exprs: &[Expression]) -> Self { - Self::new_from_to_polars(exprs) - } } #[allow(deprecated)] +#[cfg(feature = "polars")] impl DataFrameExpr for SelectExpr { /// Apply the select transformation to the lazy frame fn lazy_transformation(&self, lazy_frame: LazyFrame, _: usize) -> LazyFrame { - lazy_frame.select(&self.result_schema) + lazy_frame.select(&Vec::from_iter( + self.result_schema.iter().map(ToPolarsExpr::to_polars_expr), + )) } } @@ -52,11 +90,10 @@ impl RecordBatchExpr for SelectExpr { let easy_result: Option> = self .result_schema .iter() - .cloned() .map(|expr| match expr { - Expr::Alias(a, b) => match *a { - Expr::Column(c) if c == b => { - Some((b.to_owned(), record_batch.column_by_name(&b)?.to_owned())) + SelectTerm::AliasedResult(AliasedResultExpr { expr, alias }) => match **expr { + Expression::Column(c) if &c == alias => { + Some((c, record_batch.column_by_name(c.as_str())?.to_owned())) } _ => None, }, @@ -67,8 +104,15 @@ impl RecordBatchExpr for SelectExpr { if let Some(Ok(result)) = easy_result.map(RecordBatch::try_from_iter) { return Some(result); } - let (lazy_frame, num_input_rows) = record_batch_to_lazy_frame(record_batch)?; - #[allow(deprecated)] - lazy_frame_to_record_batch(self.lazy_transformation(lazy_frame, num_input_rows)) + #[cfg(feature = "polars")] + { + let (lazy_frame, num_input_rows) = record_batch_to_lazy_frame(record_batch)?; + #[allow(deprecated)] + lazy_frame_to_record_batch(self.lazy_transformation(lazy_frame, num_input_rows)) + } + #[cfg(not(feature = "polars"))] + { + None + } } } diff --git a/crates/proof-of-sql/src/sql/transform/slice_expr.rs b/crates/proof-of-sql/src/sql/transform/slice_expr.rs index 22e88af66..78d6f8b02 100644 --- a/crates/proof-of-sql/src/sql/transform/slice_expr.rs +++ b/crates/proof-of-sql/src/sql/transform/slice_expr.rs @@ -1,6 +1,8 @@ #[allow(deprecated)] +#[cfg(feature = "polars")] use super::DataFrameExpr; use dyn_partial_eq::DynPartialEq; +#[cfg(feature = "polars")] use polars::prelude::LazyFrame; use serde::{Deserialize, Serialize}; @@ -30,8 +32,13 @@ impl SliceExpr { } } +#[cfg(not(feature = "polars"))] +#[typetag::serde] +impl super::RecordBatchExpr for SliceExpr {} +#[cfg(feature = "polars")] super::record_batch_expr::impl_record_batch_expr_for_data_frame_expr!(SliceExpr); #[allow(deprecated)] +#[cfg(feature = "polars")] impl DataFrameExpr for SliceExpr { /// Apply the slice transformation to the given `LazyFrame`. fn lazy_transformation(&self, lazy_frame: LazyFrame, _: usize) -> LazyFrame { diff --git a/crates/proof-of-sql/src/sql/transform/test_utility.rs b/crates/proof-of-sql/src/sql/transform/test_utility.rs index d5f1c5b30..c9ee94646 100644 --- a/crates/proof-of-sql/src/sql/transform/test_utility.rs +++ b/crates/proof-of-sql/src/sql/transform/test_utility.rs @@ -1,4 +1,4 @@ -use super::*; +use super::{select_expr::SelectTerm, *}; use proof_of_sql_parser::intermediate_ast::*; pub fn lit_i64(literal: i64) -> Box { @@ -20,8 +20,9 @@ pub fn col(name: &str) -> Box { Box::new(Expression::Column(name.parse().unwrap())) } -pub(crate) fn select(result_schema: &[impl ToPolarsExpr]) -> Box { - #[allow(deprecated)] +pub(crate) fn select( + result_schema: impl IntoIterator>, +) -> Box { Box::new(SelectExpr::new(result_schema)) } @@ -34,9 +35,7 @@ pub fn schema(columns: &[(&str, &str)]) -> Vec { pub fn result(columns: &[(&str, &str)]) -> ResultExpr { let mut composition = CompositionExpr::default(); - composition.add(Box::new(SelectExpr::new_from_aliased_result_exprs( - &schema(columns), - ))); + composition.add(Box::new(SelectExpr::new(&schema(columns)))); ResultExpr::new(Box::new(composition)) }