From 828a10caf38db75e2e16b8a5eb2bbbaefab45d77 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Thu, 7 Nov 2024 22:22:10 +0100 Subject: [PATCH 1/6] template engine --- .../test/integration/postgres/sql-generation.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index f9493faba34a0..ef1d783c569ae 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -538,7 +538,7 @@ describe('SQL Generation', () => { }); `); - it('simple join', async () => { + it('simple join 1', async () => { await compiler.compile(); console.log(joinGraph.buildJoin(['visitor_checkins', 'visitors'])); From ed3d999f228ac72af84ac828e507740479b0ae67 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Fri, 22 Nov 2024 20:59:38 +0100 Subject: [PATCH 2/6] in work --- packages/cubejs-backend-shared/src/time.ts | 1 + .../src/adapter/BaseQuery.js | 43 ++- .../postgres/sql-generation.test.ts | 7 +- .../src/cube_bridge/base_tools.rs | 5 + .../src/cube_bridge/measure_definition.rs | 9 + .../cubesqlplanner/src/plan/join.rs | 72 ++++- .../cubesqlplanner/src/plan/mod.rs | 3 + .../cubesqlplanner/src/plan/query_plan.rs | 5 +- .../cubesqlplanner/src/plan/time_seria.rs | 30 ++ .../src/planner/base_measure.rs | 12 +- .../planners/multi_stage/applied_state.rs | 4 +- .../planner/planners/multi_stage/member.rs | 159 +++++++++++ .../multi_stage/member_query_planner.rs | 130 +++++++-- .../src/planner/planners/multi_stage/mod.rs | 2 + .../planners/multi_stage/query_description.rs | 18 +- .../planners/multi_stage_query_planner.rs | 258 +++++++++++++++--- .../src/planner/query_properties.rs | 17 +- .../collectors/has_cumulative_members.rs | 39 +++ .../collectors/has_multi_stage_members.rs | 13 +- .../planner/sql_evaluator/collectors/mod.rs | 2 + .../sql_evaluator/symbols/measure_symbol.rs | 12 +- .../src/planner/sql_templates/plan.rs | 69 ++--- 22 files changed, 765 insertions(+), 145 deletions(-) create mode 100644 rust/cubesqlplanner/cubesqlplanner/src/plan/time_seria.rs create mode 100644 rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs create mode 100644 rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_cumulative_members.rs diff --git a/packages/cubejs-backend-shared/src/time.ts b/packages/cubejs-backend-shared/src/time.ts index 2d795cf074bb8..e823863b21c1d 100644 --- a/packages/cubejs-backend-shared/src/time.ts +++ b/packages/cubejs-backend-shared/src/time.ts @@ -179,6 +179,7 @@ export const timeSeries = (granularity: string, dateRange: QueryDateRange, optio // moment.range works with strings const range = moment.range(dateRange[0], dateRange[1]); + console.log("!!!! timeSeries:", TIME_SERIES[granularity](range, options.timestampPrecision)); return TIME_SERIES[granularity](range, options.timestampPrecision); }; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 3ac1a2de9e600..2b39be6b7101d 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -11,7 +11,7 @@ import cronParser from 'cron-parser'; import moment from 'moment-timezone'; import inflection from 'inflection'; -import { FROM_PARTITION_RANGE, inDbTimeZone, MAX_SOURCE_ROW_LIMIT, QueryAlias, getEnv } from '@cubejs-backend/shared'; +import { FROM_PARTITION_RANGE, inDbTimeZone, MAX_SOURCE_ROW_LIMIT, QueryAlias, getEnv, timeSeries } from '@cubejs-backend/shared'; import { buildSqlAndParams as nativeBuildSqlAndParams, @@ -576,6 +576,29 @@ export class BaseQuery { return false; } + buildSqlAndParamsTest(exportAnnotatedSql) { + if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) { + if (this.externalPreAggregationQuery()) { // TODO performance + return this.externalQuery().buildSqlAndParams(exportAnnotatedSql); + } + } + const js_res = this.compilers.compiler.withQuery( + this, + () => this.cacheValue( + ['buildSqlAndParams', exportAnnotatedSql], + () => this.paramAllocator.buildSqlAndParams( + this.buildParamAnnotatedSql(), + exportAnnotatedSql, + this.shouldReuseParams + ), + { cache: this.queryCache } + ) + ); + const rust = this.buildSqlAndParamsRust(exportAnnotatedSql); + console.log('js result: ', js_res[0]); + console.log('rust result: ', rust[0]); + return js_res; + } /** * Returns a pair of SQL query string and parameter values for the query. * @param {boolean} [exportAnnotatedSql] - returns annotated sql with not rendered params if true @@ -628,6 +651,10 @@ export class BaseQuery { return res; } + //FIXME helper for native generator, maybe should be moved entire to rust + generateTimeSeries(granularity, dateRange) { + return timeSeries(granularity, dateRange); + } get shouldReuseParams() { return false; } @@ -1381,6 +1408,7 @@ export class BaseQuery { ) || undefined ); const baseQueryAlias = this.cubeAlias('base'); + console.log("!!! date join contdition: ", dateJoinCondition); const dateJoinConditionSql = dateJoinCondition.map( ([d, f]) => f( @@ -1392,6 +1420,8 @@ export class BaseQuery { ) ).join(' AND '); + console.log("!!! date join contdition sql: ", dateJoinConditionSql); + return this.overTimeSeriesSelect( cumulativeMeasures, dateSeriesSql, @@ -3232,7 +3262,16 @@ export class BaseQuery { '{% if offset is not none %}\nOFFSET {{ offset }}{% endif %}', group_by_exprs: '{{ group_by | map(attribute=\'index\') | join(\', \') }}', join: '{{ join_type }} JOIN {{ source }} ON {{ condition }}', - cte: '{{ alias }} AS ({{ query | indent(2, true) }})' + cte: '{{ alias }} AS ({{ query | indent(2, true) }})', + time_seria_select: 'SELECT date_from::timestamp AS "date_from",\n' + + 'date_to::timestamp AS "date_to" \n' + + 'FROM(\n' + + ' VALUES ' + + '{% for time_item in seria %}' + + '(\'{{ time_item | join(\'\\\', \\\'\') }}\')' + + '{% if not loop.last %}, {% endif %}' + + '{% endfor %}' + + ') AS dates (date_from, date_to)' }, expressions: { column_reference: '{% if table_name %}{{ table_name }}.{% endif %}{{ name }}', diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index ef1d783c569ae..bc27c47b4f0ba 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -538,7 +538,7 @@ describe('SQL Generation', () => { }); `); - it('simple join 1', async () => { + it('simple join', async () => { await compiler.compile(); console.log(joinGraph.buildJoin(['visitor_checkins', 'visitors'])); @@ -704,7 +704,7 @@ describe('SQL Generation', () => { }); }); - it('rolling', async () => runQueryTest({ + it('rolling 1', async () => runQueryTest({ measures: [ 'visitors.revenueRolling' ], @@ -730,7 +730,7 @@ describe('SQL Generation', () => { { visitors__created_at_day: '2017-01-10T00:00:00.000Z', visitors__revenue_rolling: null } ])); - it('rolling multiplied', async () => runQueryTest({ + it('rolling multiplied 1', async () => runQueryTest({ measures: [ 'visitors.revenueRolling', 'visitor_checkins.visitor_checkins_count' @@ -916,6 +916,7 @@ describe('SQL Generation', () => { { visitors__created_at_day: '2017-01-10T00:00:00.000Z', visitors__running_revenue_per_count: '300' } ])); + it('hll rolling (BigQuery)', async () => { await compiler.compile(); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs index d5aef3c6d812d..5ade3438edc5c 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs @@ -38,4 +38,9 @@ pub trait BaseTools { fn filter_group_function(&self) -> Result, CubeError>; fn timestamp_precision(&self) -> Result; fn in_db_time_zone(&self, date: String) -> Result; + fn generate_time_series( + &self, + granularity: String, + date_range: Vec, + ) -> Result>, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs index 5b23f4e971e4e..1f5c55e95b363 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs @@ -21,6 +21,13 @@ pub struct TimeShiftReference { pub time_dimension: String, } +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] +pub struct RollingWindow { + trailing: Option, + leading: Option, + offset: Option, +} + #[derive(Serialize, Deserialize, Debug)] pub struct MeasureDefinitionStatic { #[serde(rename = "type")] @@ -37,6 +44,8 @@ pub struct MeasureDefinitionStatic { pub group_by_references: Option>, #[serde(rename = "timeShiftReferences")] pub time_shift_references: Option>, + #[serde(rename = "rollingWindow")] + pub rolling_window: Option, } #[nativebridge::native_bridge(MeasureDefinitionStatic)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs index 19e8283bc5e68..21431aca45372 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs @@ -1,10 +1,80 @@ -use super::{Schema, SingleAliasedSource}; +use super::{time_seria, Schema, SingleAliasedSource}; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::{BaseJoinCondition, BaseMember, VisitorContext}; use cubenativeutils::CubeError; use std::rc::Rc; +pub struct RollingWindowJoinCondition { + tailing_interval: Option, + leading_interval: Option, + offset: String, + is_from_start_to_end: bool, + time_dimension: Vec>, +} + +impl RollingWindowJoinCondition { + pub fn new( + trailing_interval: Option, + leading_interval: Option, + offset: String, + is_from_start_to_end: bool, + dimensions: Vec>, + ) -> Self { + Self { + tailing_interval, + leading_interval, + offset, + is_from_start_to_end, + time_dimension, + } + } + + pub fn to_sql( + &self, + templates: &PlanSqlTemplates, + context: Rc, + schema: Rc, + ) -> Result { + let result = if self.dimensions.is_empty() { + format!("1 = 1") + } else { + let conditions = vec![]; + self.dimensions + .iter() + .map(|dim| -> Result { + if let Some(trailing_interval) = self.trailing_interval { + if tailing_interval == "unbounded" { + let seria_column = "date_from", + } + } + + + }) + .collect::, _>>()? + .join(" AND ") + }; + Ok(result) + } + + fn resolve_member_alias( + &self, + templates: &PlanSqlTemplates, + context: Rc, + source: &String, + dimension: &Rc, + schema: Rc, + ) -> Result { + let schema = schema.extract_source_schema(source); + let source = Some(source.clone()); + if let Some(column) = schema.find_column_for_member(&dimension.full_name(), &source) { + templates.column_reference(&source, &column.alias.clone()) + } else { + dimension.to_sql(context.clone(), schema.clone()) + } + } +} + pub struct DimensionJoinCondition { left_source: String, right_source: String, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs index 5d98909183278..5438fe4bd04f0 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs @@ -8,6 +8,7 @@ pub mod order; pub mod query_plan; pub mod schema; pub mod select; +pub mod time_seria; pub mod union; pub use builder::{JoinBuilder, SelectBuilder}; @@ -20,4 +21,6 @@ pub use order::OrderBy; pub use query_plan::QueryPlan; pub use schema::{Schema, SchemaColumn, SchemaCube}; pub use select::{AliasedExpr, Select}; +pub use time_seria::TimeSeria; pub use union::Union; + diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/query_plan.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/query_plan.rs index e8224fb74aebf..09d46417ed9e3 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/query_plan.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/query_plan.rs @@ -1,4 +1,4 @@ -use super::{Schema, Select, Union}; +use super::{Schema, Select, TimeSeria, Union}; use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; use std::rc::Rc; @@ -6,6 +6,7 @@ use std::rc::Rc; pub enum QueryPlan { Select(Rc), Union(Rc), - TimeSeria(Rc), + TimeSeries(Rc), } impl QueryPlan { @@ -14,14 +14,14 @@ impl QueryPlan { match self { QueryPlan::Select(select) => select.make_schema(self_alias), QueryPlan::Union(union) => union.make_schema(self_alias), - QueryPlan::TimeSeria(seria) => seria.make_schema(self_alias), + QueryPlan::TimeSeries(series) => series.make_schema(self_alias), } } pub fn to_sql(&self, templates: &PlanSqlTemplates) -> Result { match self { QueryPlan::Select(s) => s.to_sql(templates), QueryPlan::Union(u) => u.to_sql(templates), - QueryPlan::TimeSeria(seria) => seria.to_sql(templates), + QueryPlan::TimeSeries(series) => series.to_sql(templates), } } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/time_seria.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs similarity index 78% rename from rust/cubesqlplanner/cubesqlplanner/src/plan/time_seria.rs rename to rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs index 28065d3fad6d9..b2c891bac287a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/time_seria.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs @@ -3,25 +3,26 @@ use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; use std::rc::Rc; -pub struct TimeSeria { +pub struct TimeSeries { pub time_dimension_name: String, pub from_date: Option, pub to_date: Option, pub seria: Vec>, } -impl TimeSeria { +impl TimeSeries { pub fn make_schema(&self, self_alias: Option) -> Schema { - let column = SchemaColumn::new( + /* let column = SchemaColumn::new( self_alias, format!("from_date"), self.time_dimension_name.clone(), ); - Schema::new(vec![column], vec![]) + Schema::new(vec![column], vec![]) */ + Schema::empty() } pub fn to_sql(&self, templates: &PlanSqlTemplates) -> Result { - templates.time_seria_select( + templates.time_series_select( self.from_date.clone(), self.to_date.clone(), self.seria.clone(), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs index a665950fea600..6bb827a7c8c2e 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs @@ -1,5 +1,6 @@ use crate::cube_bridge::measure_definition::{MeasureDefinition, TimeShiftReference}; use crate::planner::sql_evaluator::EvaluationNode; +use crate::planner::BaseMember; use crate::planner::BaseTimeDimension; use cubenativeutils::CubeError; use lazy_static::lazy_static; @@ -61,7 +62,15 @@ impl MultiStageTimeShift { #[derive(Clone)] pub enum MultiStageLeafMemberType { Measure, - TimeSeria(Rc), + TimeSeries(Rc), +} + +#[derive(Clone)] +pub struct RollingWindowDescription { + pub time_dimension: Rc, + pub trailing: Option, + pub leading: Option, + pub offset: String, } #[derive(Clone)] @@ -69,7 +78,7 @@ pub enum MultiStageInodeMemberType { Rank, Aggregate, Calculate, - RollingWindow, + RollingWindow(RollingWindowDescription), } #[derive(Clone)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index 87da9b6ae74dd..a2d73ee0bda9e 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -1,10 +1,10 @@ use super::{ MultiStageInodeMember, MultiStageInodeMemberType, MultiStageMemberType, - MultiStageQueryDescription, + MultiStageQueryDescription, RollingWindowDescription, }; use crate::plan::{ Cte, Expr, FilterGroup, FilterItem, From, JoinBuilder, JoinCondition, MemberExpression, - OrderBy, QueryPlan, Schema, SelectBuilder, TimeSeria, + OrderBy, QueryPlan, Schema, SelectBuilder, TimeSeries, }; use crate::planner::planners::{ FullKeyAggregateQueryPlanner, MultipliedMeasuresQueryPlanner, OrderPlanner, SimpleQueryPlanner, @@ -43,17 +43,22 @@ impl MultiStageMemberQueryPlanner { cte_schemas: &HashMap>, ) -> Result, CubeError> { match self.description.member().member_type() { - MultiStageMemberType::Inode(member) => self.plan_for_cte_query(member, cte_schemas), + MultiStageMemberType::Inode(member) => match member.inode_type() { + MultiStageInodeMemberType::RollingWindow(rolling_window_desc) => { + self.plan_rolling_window_query(rolling_window_desc, member, cte_schemas) + } + _ => self.plan_for_cte_query(member, cte_schemas), + }, MultiStageMemberType::Leaf(node) => match node { super::MultiStageLeafMemberType::Measure => self.plan_for_leaf_cte_query(), - super::MultiStageLeafMemberType::TimeSeria(time_dimension) => { - self.plan_time_seria_query(time_dimension.clone()) + super::MultiStageLeafMemberType::TimeSeries(time_dimension) => { + self.plan_time_series_query(time_dimension.clone()) } }, } } - fn plan_time_seria_query( + fn plan_time_series_query( &self, time_dimension: Rc, ) -> Result, CubeError> { @@ -65,31 +70,78 @@ impl MultiStageMemberQueryPlanner { .query_tools .base_tools() .generate_time_series(granularity, date_range.clone())?; - let time_seira = TimeSeria { + let time_seira = TimeSeries { time_dimension_name: time_dimension.full_name(), from_date: Some(from_date), to_date: Some(to_date), seria, }; - let query_plan = Rc::new(QueryPlan::TimeSeria(Rc::new(time_seira))); - Ok(Rc::new(Cte::new(query_plan, format!("time_seria")))) + let query_plan = Rc::new(QueryPlan::TimeSeries(Rc::new(time_seira))); + Ok(Rc::new(Cte::new(query_plan, format!("time_series")))) } fn plan_rolling_window_query( &self, + rolling_window_desc: &RollingWindowDescription, multi_stage_member: &MultiStageInodeMember, cte_schemas: &HashMap>, ) -> Result, CubeError> { + let inputs = self.input_cte_aliases(); let dimensions = self.all_dimensions(); - let root_alias = format!("time_seria"); - + let root_alias = format!("time_series"); let cte_schema = cte_schemas.get(&inputs[0]).unwrap().clone(); + let mut join_builder = JoinBuilder::new_from_table_reference( inputs[0].clone(), cte_schema, Some(root_alias.clone()), ); + + for (i, input) in inputs.iter().skip(1).enumerate() { + let alias = format!("rolling_{}", i + 1); + let on = JoinCondition::new_rolling_join( + alias.clone(), + root_alias.clone(), + rolling_window_desc.trailing.clone(), + rolling_window_desc.leading.clone(), + rolling_window_desc.offset.clone(), + rolling_window_desc.time_dimension.clone(), + ); + let cte_schema = cte_schemas.get(input).unwrap().clone(); + join_builder.inner_join_table_reference( + input.clone(), + cte_schema, + Some(format!("rolling_{}", i + 1)), + on, + ); + } + + let from = From::new_from_join(join_builder.build()); + + let group_by = dimensions + .iter() + .map(|dim| Expr::Member(MemberExpression::new(dim.clone(), None))) + .collect_vec(); + + let context_factory = SqlNodesFactory::new(); + let node_context = context_factory.default_node_processor(); + + let mut select_builder = SelectBuilder::new(from, VisitorContext::new(None, node_context)); + for dim in dimensions.iter() { + select_builder.add_projection_member(&dim, None, None); + } + + let query_member = self.query_member_as_base_member()?; + select_builder.add_projection_member(&query_member, None, None); + select_builder.set_group_by(group_by); + select_builder.set_order_by(self.query_order()?); + let select = select_builder.build(); + + Ok(Rc::new(Cte::new_from_select( + Rc::new(select), + self.description.alias().clone(), + ))) } fn plan_for_cte_query( @@ -221,7 +273,6 @@ impl MultiStageMemberQueryPlanner { let allowed_filter_members = self.description.state().allowed_filter_members().clone(); - println!("!!!! meas count {}", measures.len()); let cte_query_properties = QueryProperties::try_new_from_precompiled( self.query_tools.clone(), measures, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs index 3e6dd34e5b706..d5b8e965d19b7 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs @@ -1,7 +1,7 @@ use super::multi_stage::{ MultiStageAppliedState, MultiStageInodeMember, MultiStageInodeMemberType, MultiStageLeafMemberType, MultiStageMember, MultiStageMemberQueryPlanner, MultiStageMemberType, - MultiStageQueryDescription, MultiStageTimeShift, + MultiStageQueryDescription, MultiStageTimeShift, RollingWindowDescription, }; use crate::plan::{Cte, From, Schema, Select, SelectBuilder}; use crate::planner::query_tools::QueryTools; @@ -9,8 +9,8 @@ use crate::planner::sql_evaluator::collectors::has_multi_stage_members; use crate::planner::sql_evaluator::collectors::member_childs; use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory; use crate::planner::sql_evaluator::EvaluationNode; -use crate::planner::QueryProperties; use crate::planner::{BaseDimension, BaseMeasure, VisitorContext}; +use crate::planner::{BaseTimeDimension, QueryProperties}; use cubenativeutils::CubeError; use itertools::Itertools; use std::collections::HashMap; @@ -151,35 +151,29 @@ impl MultiStageQueryPlanner { Ok(inode) } - fn add_time_seria( + fn add_time_series( &self, + time_dimension: Rc, state: Rc, descriptions: &mut Vec>, ) -> Result, CubeError> { let description = - if let Some(description) = descriptions.iter().find(|d| d.alias() == "time_seria") { + if let Some(description) = descriptions.iter().find(|d| d.alias() == "time_series") { description.clone() } else { - let time_dimensions = self.query_properties.time_dimensions(); - if time_dimensions.len() != 1 { - return Err(CubeError::internal( - "Rolling window requires one time dimension".to_string(), - )); - } - let time_dimension = time_dimensions[0].clone(); - let time_seria_node = MultiStageQueryDescription::new( + let time_series_node = MultiStageQueryDescription::new( MultiStageMember::new( - MultiStageMemberType::Leaf(MultiStageLeafMemberType::TimeSeria( + MultiStageMemberType::Leaf(MultiStageLeafMemberType::TimeSeries( time_dimension.clone(), )), time_dimension.member_evaluator(), ), state.clone(), vec![], - "time_seria".to_string(), + "time_series".to_string(), ); - descriptions.push(time_seria_node.clone()); - time_seria_node + descriptions.push(time_series_node.clone()); + time_series_node }; Ok(description) } @@ -194,7 +188,7 @@ impl MultiStageQueryPlanner { let description = MultiStageQueryDescription::new( MultiStageMember::new( MultiStageMemberType::Leaf(MultiStageLeafMemberType::Measure), - member.clone(), + member, ), state.clone(), vec![], @@ -212,17 +206,35 @@ impl MultiStageQueryPlanner { ) -> Result>, CubeError> { if let Some(measure) = BaseMeasure::try_new(member.clone(), self.query_tools.clone())? { if let Some(rolling_window) = measure.rolling_window() { - self.add_time_seria(state.clone(), descriptions)?; - let input = vec![self.add_rolling_window_base( - member.clone(), - state.clone(), - descriptions, - )?]; + let time_dimensions = self.query_properties.time_dimensions(); + if time_dimensions.len() != 1 { + return Err(CubeError::internal( + "Rolling window requires one time dimension".to_string(), + )); + } + let time_dimension = time_dimensions[0].clone(); + let source_name = format!("_{}_base", member.name()); + let (member, source) = member.try_split_measure(source_name).unwrap(); //FIXME + //unwrap!!! + + let input = vec![ + self.add_time_series(time_dimension.clone(), state.clone(), descriptions)?, + self.add_rolling_window_base(source.clone(), state.clone(), descriptions)?, + ]; + + let time_dimension = time_dimensions[0].clone(); let alias = format!("cte_{}", descriptions.len()); + let rolling_window_descr = RollingWindowDescription { + time_dimension: time_dimension.clone(), + trailing: rolling_window.trailing.clone(), + leading: rolling_window.leading.clone(), + offset: rolling_window.offset.clone().unwrap_or("end".to_string()), + }; + let inode_member = MultiStageInodeMember::new( - MultiStageInodeMemberType::RollingWindow, + MultiStageInodeMemberType::RollingWindow(rolling_window_descr), vec![], vec![], None, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index 0ae24e1e5f2ff..63465f24c8ef7 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -390,7 +390,7 @@ impl QueryProperties { Ok(true) } - pub fn should_use_time_seria(&self) -> Result { + pub fn should_use_time_series(&self) -> Result { for member in self.all_members(false) { if has_cumulative_members(&member.member_evaluator())? { return Ok(true); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs index a9ac62c42aba6..635c7e944c076 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs @@ -5,6 +5,7 @@ use cubenativeutils::CubeError; use std::collections::HashMap; use std::rc::Rc; +#[derive(Clone)] pub struct StructDependency { pub sql_fn: Option>, pub to_string_fn: Option>, @@ -25,12 +26,14 @@ impl StructDependency { } } +#[derive(Clone)] pub enum ContextSymbolDep { SecurityContext, FilterParams, FilterGroup, } +#[derive(Clone)] pub enum Dependency { SingleDependency(Rc), StructDependency(StructDependency), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs index 02355626a2c28..5c53c7833ca61 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs @@ -62,10 +62,33 @@ impl EvaluationNode { self.symbol.full_name() } + pub fn name(&self) -> String { + self.symbol.name() + } + pub fn is_measure(&self) -> bool { self.symbol.is_measure() } pub fn is_dimension(&self) -> bool { self.symbol.is_dimension() } + + pub fn try_split_measure(self: Rc, source_name: String) -> Option<(Rc, Rc)> { + match &self.symbol { + MemberSymbolType::Measure(measure_symbol) => { + let (measure, source) = measure_symbol.split_with_source(source_name); + let source = Self { + symbol: MemberSymbolType::Measure(source), + deps: self.deps.clone(), + }; + let source = Rc::new(source); + let measure = Self { + symbol: MemberSymbolType::Measure(measure), + deps: vec![Dependency::SingleDependency(source.clone())], + }; + Some((Rc::new(measure), source)) + } + _ => None, + } + } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs index 0acd90c5f868d..17209f5af693c 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs @@ -1,4 +1,5 @@ use super::SqlNode; +use crate::cube_bridge::memeber_sql::MemberSqlArg; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; @@ -25,7 +26,23 @@ impl SqlNode for EvaluateSqlNode { let args = visitor.evaluate_deps(node, node_processor.clone())?; match node.symbol() { MemberSymbolType::Dimension(ev) => ev.evaluate_sql(args), - MemberSymbolType::Measure(ev) => ev.evaluate_sql(args), + MemberSymbolType::Measure(ev) => { + let res = if ev.is_splitted_source() { + //FIXME hack for working with + //measures like rolling window + if !args.is_empty() { + match &args[0] { + MemberSqlArg::String(s) => s.clone(), + _ => "".to_string(), + } + } else { + "".to_string() + } + } else { + ev.evaluate_sql(args)? + }; + Ok(res) + } MemberSymbolType::CubeTable(ev) => ev.evaluate_sql(args), MemberSymbolType::CubeName(ev) => ev.evaluate_sql(args), MemberSymbolType::SimpleSql(ev) => ev.evaluate_sql(args), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs index f0fa203341246..be6d0b60603c6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs @@ -8,6 +8,7 @@ use crate::planner::sql_evaluator::{Compiler, Dependency, EvaluationNode}; use cubenativeutils::CubeError; use std::rc::Rc; +#[derive(Clone)] pub struct MeasureOrderBy { evaluation_node: Rc, direction: String, @@ -30,6 +31,7 @@ impl MeasureOrderBy { } } +#[derive(Clone)] pub struct MeasureSymbol { cube_name: String, name: String, @@ -37,6 +39,7 @@ pub struct MeasureSymbol { measure_filters: Vec>, measure_order_by: Vec, member_sql: Rc, + is_splitted_source: bool, } impl MeasureSymbol { @@ -55,6 +58,7 @@ impl MeasureSymbol { definition, measure_filters, measure_order_by, + is_splitted_source: false, } } @@ -62,6 +66,24 @@ impl MeasureSymbol { format!("{}.{}", self.cube_name, self.name) } + pub fn is_splitted_source(&self) -> bool { + self.is_splitted_source + } + + pub fn split_with_source(&self, source_name: String) -> (Self, Self) { + let mut measure_with_source = self.clone(); + measure_with_source.is_splitted_source = true; + let source = Self::new( + self.cube_name.clone(), + source_name.clone(), + self.member_sql.clone(), + self.definition().clone(), + self.measure_filters.clone(), + self.measure_order_by.clone(), + ); + (measure_with_source, source) + } + pub fn is_calculated(&self) -> bool { match self.definition.static_data().measure_type.as_str() { "number" | "string" | "time" | "boolean" => true, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs index b8137abda10cb..d8d3ce1836ce1 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs @@ -1,3 +1,5 @@ +use std::rc::Rc; + pub trait MemberSymbol { fn cube_name(&self) -> &String; fn name(&self) -> &String; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs index 830c8e7f4e367..d719fb93492c2 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs @@ -1,4 +1,6 @@ -use super::{CubeNameSymbol, CubeTableSymbol, DimensionSymbol, MeasureSymbol, SimpleSqlSymbol}; +use super::{ + CubeNameSymbol, CubeTableSymbol, DimensionSymbol, MeasureSymbol, MemberSymbol, SimpleSqlSymbol, +}; pub enum MemberSymbolType { Dimension(DimensionSymbol), Measure(MeasureSymbol), @@ -17,6 +19,15 @@ impl MemberSymbolType { MemberSymbolType::SimpleSql(_) => "".to_string(), } } + pub fn name(&self) -> String { + match self { + MemberSymbolType::Dimension(d) => d.name().clone(), + MemberSymbolType::Measure(m) => m.name().clone(), + MemberSymbolType::CubeName(c) => c.cube_name().clone(), + MemberSymbolType::CubeTable(c) => c.cube_name().clone(), + MemberSymbolType::SimpleSql(_) => "".to_string(), + } + } pub fn is_measure(&self) -> bool { matches!(self, Self::Measure(_)) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs index 23af4cc2e063a..26b12acacd25a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs @@ -121,14 +121,14 @@ impl PlanSqlTemplates { ) } - pub fn time_seria_select( + pub fn time_series_select( &self, from_date: Option, to_date: Option, seria: Vec>, ) -> Result { self.render.render_template( - "statements/time_seria_select", + "statements/time_series_select", context! { from_date => from_date, to_date => to_date, From e597ab357c9555c2a4ba6d12fa622ff0cb457ff3 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Tue, 3 Dec 2024 16:55:23 +0100 Subject: [PATCH 4/6] multi stage and running total pass tests --- packages/cubejs-backend-shared/src/time.ts | 1 - .../src/adapter/BaseQuery.js | 4 +- .../postgres/sql-generation.test.ts | 12 +- .../cubesqlplanner/src/plan/filter.rs | 10 +- .../cubesqlplanner/src/plan/join.rs | 93 ++++----- .../cubesqlplanner/src/plan/time_series.rs | 7 +- .../src/planner/base_measure.rs | 8 + .../src/planner/base_time_dimension.rs | 24 ++- .../src/planner/filter/base_filter.rs | 75 +++++++- .../src/planner/filter/compiler.rs | 6 +- .../src/planner/filter/filter_operator.rs | 3 + .../cubesqlplanner/src/planner/filter/mod.rs | 1 + .../src/planner/granularity_helper.rs | 149 ++++++++++++++ .../cubesqlplanner/src/planner/mod.rs | 2 + .../planners/multi_stage/applied_state.rs | 181 ++++++++++++++++-- .../planner/planners/multi_stage/member.rs | 6 + .../multi_stage/member_query_planner.rs | 47 ++--- .../planners/multi_stage_query_planner.rs | 73 ++++++- .../collectors/has_multi_stage_members.rs | 4 +- .../sql_node_transformers/set_schema.rs | 12 +- .../sql_evaluator/sql_nodes/evaluate_sql.rs | 18 +- .../sql_evaluator/sql_nodes/factory.rs | 22 ++- .../sql_evaluator/sql_nodes/final_measure.rs | 6 +- .../planner/sql_evaluator/sql_nodes/mod.rs | 2 + .../sql_evaluator/sql_nodes/rolling_window.rs | 76 ++++++++ .../sql_evaluator/symbols/measure_symbol.rs | 8 + .../src/planner/sql_templates/filter.rs | 21 ++ 27 files changed, 711 insertions(+), 160 deletions(-) create mode 100644 rust/cubesqlplanner/cubesqlplanner/src/planner/granularity_helper.rs create mode 100644 rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs diff --git a/packages/cubejs-backend-shared/src/time.ts b/packages/cubejs-backend-shared/src/time.ts index e823863b21c1d..2d795cf074bb8 100644 --- a/packages/cubejs-backend-shared/src/time.ts +++ b/packages/cubejs-backend-shared/src/time.ts @@ -179,7 +179,6 @@ export const timeSeries = (granularity: string, dateRange: QueryDateRange, optio // moment.range works with strings const range = moment.range(dateRange[0], dateRange[1]); - console.log("!!!! timeSeries:", TIME_SERIES[granularity](range, options.timestampPrecision)); return TIME_SERIES[granularity](range, options.timestampPrecision); }; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 30787de1b7cc5..d325eb455cc73 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -734,7 +734,6 @@ export class BaseQuery { offset = offset || 'end'; return this.timeDimensions.map( d => [d, (dateFrom, dateTo, dateField, dimensionDateFrom, dimensionDateTo, isFromStartToEnd) => { - console.log("!!!!!! IsFromStartToEnd: ", isFromStartToEnd, " --"); // dateFrom based window const conditions = []; if (trailingInterval !== 'unbounded') { @@ -1420,7 +1419,6 @@ export class BaseQuery { ) ).join(' AND '); - console.log("!!! date join contdition sql: ", dateJoinConditionSql); return this.overTimeSeriesSelect( cumulativeMeasures, @@ -3292,6 +3290,8 @@ export class BaseQuery { cube: 'CUBE({{ exprs_concat }})', negative: '-({{ expr }})', not: 'NOT ({{ expr }})', + add_interval: '{{ date }} + interval \'{{ interval }}\'', + sub_interval: '{{ date }} - interval \'{{ interval }}\'', true: 'TRUE', false: 'FALSE', like: '{{ expr }} {% if negated %}NOT {% endif %}LIKE {{ pattern }}', diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index bc27c47b4f0ba..103a9f6dbee12 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -64,7 +64,7 @@ describe('SQL Generation', () => { offset: 'start' } }, - revenueRolling3day: { + revenueRollingThreeDay: { type: 'sum', sql: 'amount', rollingWindow: { @@ -666,7 +666,7 @@ describe('SQL Generation', () => { console.log(query.buildSqlAndParams()); // TODO ordering doesn't work for running total - return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + return dbRunner.testQuery(query.buildSqlAndParamsTest()).then(res => { console.log(JSON.stringify(res)); expect(res).toEqual( [{ @@ -704,7 +704,7 @@ describe('SQL Generation', () => { }); }); - it('rolling 1', async () => runQueryTest({ + it('rolling', async () => runQueryTest({ measures: [ 'visitors.revenueRolling' ], @@ -730,7 +730,7 @@ describe('SQL Generation', () => { { visitors__created_at_day: '2017-01-10T00:00:00.000Z', visitors__revenue_rolling: null } ])); - it('rolling multiplied 1', async () => runQueryTest({ + it('rolling multiplied', async () => runQueryTest({ measures: [ 'visitors.revenueRolling', 'visitor_checkins.visitor_checkins_count' @@ -769,7 +769,7 @@ describe('SQL Generation', () => { it('rolling month', async () => runQueryTest({ measures: [ - 'visitors.revenueRolling3day' + 'visitors.revenueRollingThreeDay' ], timeDimensions: [{ dimension: 'visitors.created_at', @@ -781,7 +781,7 @@ describe('SQL Generation', () => { }], timezone: 'America/Los_Angeles' }, [ - { visitors__created_at_week: '2017-01-09T00:00:00.000Z', visitors__revenue_rolling3day: '900' } + { visitors__created_at_week: '2017-01-09T00:00:00.000Z', visitors__revenue_rolling_three_day: '900' } ])); it('rolling count', async () => runQueryTest({ diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs index f118aefe5d8e5..9776ad592a37a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs @@ -6,7 +6,7 @@ use cubenativeutils::CubeError; use std::fmt; use std::rc::Rc; -#[derive(Clone)] +#[derive(Clone, PartialEq)] pub enum FilterGroupOperator { Or, And, @@ -18,13 +18,19 @@ pub struct FilterGroup { pub items: Vec, } +impl PartialEq for FilterGroup { + fn eq(&self, other: &Self) -> bool { + self.operator == other.operator && self.items == other.items + } +} + impl FilterGroup { pub fn new(operator: FilterGroupOperator, items: Vec) -> Self { Self { operator, items } } } -#[derive(Clone)] +#[derive(Clone, PartialEq)] pub enum FilterItem { Group(Rc), Item(Rc), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs index 0d06cb2cf33b9..bd32f03c2b825 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs @@ -2,6 +2,7 @@ use super::{time_series, Schema, SingleAliasedSource}; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::{BaseJoinCondition, BaseMember, VisitorContext}; use cubenativeutils::CubeError; +use lazy_static::lazy_static; use std::rc::Rc; @@ -33,29 +34,6 @@ impl RollingWindowJoinCondition { } } - /* - * - offset = offset || 'end'; - return this.timeDimensions.map( - d => [d, (dateFrom, dateTo, dateField, dimensionDateFrom, dimensionDateTo, isFromStartToEnd) => { - // dateFrom based window - const conditions = []; - if (trailingInterval !== 'unbounded') { - const startDate = isFromStartToEnd || offset === 'start' ? dateFrom : dateTo; - const trailingStart = trailingInterval ? this.subtractInterval(startDate, trailingInterval) : startDate; - const sign = offset === 'start' ? '>=' : '>'; - conditions.push(`${dateField} ${sign} ${trailingStart}`); - } - if (leadingInterval !== 'unbounded') { - const endDate = isFromStartToEnd || offset === 'end' ? dateTo : dateFrom; - const leadingEnd = leadingInterval ? this.addInterval(endDate, leadingInterval) : endDate; - const sign = offset === 'end' ? '<=' : '<'; - conditions.push(`${dateField} ${sign} ${leadingEnd}`); - } - return conditions.length ? conditions.join(' AND ') : '1 = 1'; - }] - ); - */ pub fn to_sql( &self, templates: &PlanSqlTemplates, @@ -63,55 +41,48 @@ impl RollingWindowJoinCondition { schema: Rc, ) -> Result { let mut conditions = vec![]; - /* let date_column_alias = if let Some(column) = schema.find_column_for_member(&self.time_dimension.full_name(), &None) { - templates.column_reference(&source, &column.alias.clone()) - } else { - dimension.to_sql(context.clone(), schema.clone()) - } */ let date_column_alias = self.resolve_time_column_alias(templates, context.clone(), schema.clone())?; - if let Some(trailing_interval) = &self.trailing_interval { - if trailing_interval != "unbounded" { - let start_date = if self.offset == "start" { - templates - .column_reference(&Some(self.time_series_source.clone()), "date_from")? - } else { - templates.column_reference(&Some(self.time_series_source.clone()), "date_to")? - }; - let trailing_start = if let Some(trailing_interval) = &self.trailing_interval { - format!("{start_date} - interval '{trailing_interval}'") - } else { - start_date - }; + lazy_static! { + static ref UNBOUNDED: Option = Some("unbounded".to_string()); + } - let sign = if self.offset == "start" { ">=" } else { ">" }; + if self.trailing_interval != *UNBOUNDED { + let start_date = if self.offset == "start" { + templates.column_reference(&Some(self.time_series_source.clone()), "date_from")? + } else { + templates.column_reference(&Some(self.time_series_source.clone()), "date_to")? + }; - conditions.push(format!("{date_column_alias} {sign} {trailing_start}")); - } + let trailing_start = if let Some(trailing_interval) = &self.trailing_interval { + format!("{start_date} - interval '{trailing_interval}'") + } else { + start_date + }; + + let sign = if self.offset == "start" { ">=" } else { ">" }; + + conditions.push(format!("{date_column_alias} {sign} {trailing_start}")); } - if let Some(leading_interval) = &self.trailing_interval { - if leading_interval != "unbounded" { - let end_date = if self.offset == "end" { - templates.column_reference(&Some(self.time_series_source.clone()), "date_to")? - } else { - templates - .column_reference(&Some(self.time_series_source.clone()), "date_from")? - }; + if self.leading_interval != *UNBOUNDED { + let end_date = if self.offset == "end" { + templates.column_reference(&Some(self.time_series_source.clone()), "date_to")? + } else { + templates.column_reference(&Some(self.time_series_source.clone()), "date_from")? + }; - let leading_end = if let Some(leading_interval) = &self.leading_interval { - format!("{end_date} + interval '{leading_interval}'") - } else { - end_date - }; + let leading_end = if let Some(leading_interval) = &self.leading_interval { + format!("{end_date} + interval '{leading_interval}'") + } else { + end_date + }; - let sign = if self.offset == "end" { "<=" } else { "<" }; + let sign = if self.offset == "end" { "<=" } else { "<" }; - conditions.push(format!("{date_column_alias} {sign} {leading_end}")); - } + conditions.push(format!("{date_column_alias} {sign} {leading_end}")); } - let result = if conditions.is_empty() { templates.always_true()? } else { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs index b2c891bac287a..774fa0e825878 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs @@ -12,13 +12,12 @@ pub struct TimeSeries { impl TimeSeries { pub fn make_schema(&self, self_alias: Option) -> Schema { - /* let column = SchemaColumn::new( + let column = SchemaColumn::new( self_alias, - format!("from_date"), + format!("date_from"), self.time_dimension_name.clone(), ); - Schema::new(vec![column], vec![]) */ - Schema::empty() + Schema::new(vec![column], vec![]) } pub fn to_sql(&self, templates: &PlanSqlTemplates) -> Result { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_measure.rs index 66b4f263483f8..5fa5ce929c703 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_measure.rs @@ -205,6 +205,14 @@ impl BaseMeasure { self.rolling_window().is_some() } + pub fn is_running_total(&self) -> bool { + self.measure_type() == "runningTotal" + } + + pub fn is_cumulative(&self) -> bool { + self.is_rolling_window() || self.is_running_total() + } + //FIXME dublicate with symbol pub fn measure_type(&self) -> &String { &self.definition.static_data().measure_type diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs index 38fa14ff5f7e7..f6e297971e249 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs @@ -11,6 +11,7 @@ pub struct BaseTimeDimension { query_tools: Rc, granularity: Option, date_range: Option>, + alias_suffix: String, } impl BaseMember for BaseTimeDimension { @@ -55,12 +56,7 @@ impl BaseMember for BaseTimeDimension { } fn alias_suffix(&self) -> Option { - let granularity = if let Some(granularity) = &self.granularity { - granularity - } else { - "day" - }; - Some(granularity.to_string()) + Some(self.alias_suffix.clone()) } } @@ -71,14 +67,30 @@ impl BaseTimeDimension { granularity: Option, date_range: Option>, ) -> Result, CubeError> { + let alias_suffix = if let Some(granularity) = &granularity { + granularity.clone() + } else { + "day".to_string() + }; Ok(Rc::new(Self { dimension: BaseDimension::try_new_required(member_evaluator, query_tools.clone())?, query_tools, granularity, date_range, + alias_suffix, })) } + pub fn change_granularity(&self, new_granularity: Option) -> Rc { + Rc::new(Self { + dimension: self.dimension.clone(), + query_tools: self.query_tools.clone(), + granularity: new_granularity, + date_range: self.date_range.clone(), + alias_suffix: self.alias_suffix.clone(), + }) + } + pub fn get_granularity(&self) -> Option { self.granularity.clone() } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs index 02247f982135e..fb1de9080e192 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs @@ -26,6 +26,14 @@ pub struct BaseFilter { templates: FilterTemplates, } +impl PartialEq for BaseFilter { + fn eq(&self, other: &Self) -> bool { + self.filter_type == other.filter_type + && self.filter_operator == other.filter_operator + && self.values == other.values + } +} + lazy_static! { static ref DATE_TIME_LOCAL_MS_RE: Regex = Regex::new(r"^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\d$").unwrap(); @@ -39,7 +47,7 @@ impl BaseFilter { query_tools: Rc, member_evaluator: Rc, filter_type: FilterType, - filter_operator: String, + filter_operator: FilterOperator, values: Option>>, ) -> Result, CubeError> { let templates = FilterTemplates::new(query_tools.templates_render()); @@ -52,12 +60,35 @@ impl BaseFilter { query_tools, member_evaluator, filter_type, - filter_operator: FilterOperator::from_str(&filter_operator)?, + filter_operator, values, templates, })) } + pub fn change_operator( + &self, + filter_operator: FilterOperator, + values: Vec>, + ) -> Rc { + Rc::new(Self { + query_tools: self.query_tools.clone(), + member_evaluator: self.member_evaluator.clone(), + filter_type: self.filter_type.clone(), + filter_operator, + values, + templates: self.templates.clone(), + }) + } + + pub fn values(&self) -> &Vec> { + &self.values + } + + pub fn filter_operator(&self) -> &FilterOperator { + &self.filter_operator + } + pub fn member_name(&self) -> String { self.member_evaluator.full_name() } @@ -77,6 +108,7 @@ impl BaseFilter { FilterOperator::Equal => self.equals_where(&member_sql)?, FilterOperator::NotEqual => self.not_equals_where(&member_sql)?, FilterOperator::InDateRange => self.in_date_range(&member_sql)?, + FilterOperator::InDateRangeExtended => self.in_date_range_extended(&member_sql)?, FilterOperator::In => self.in_where(&member_sql)?, FilterOperator::NotIn => self.not_in_where(&member_sql)?, FilterOperator::Set => self.set_where(&member_sql)?, @@ -127,6 +159,45 @@ impl BaseFilter { .time_range_filter(member_sql.to_string(), from, to) } + fn extend_date_range_bound( + &self, + date: String, + interval: &Option, + is_sub: bool, + ) -> Result { + if let Some(interval) = interval { + if interval != "unbounded" { + if is_sub { + self.templates.sub_interval(date, interval.clone()) + } else { + self.templates.add_interval(date, interval.clone()) + } + } else { + Ok(date.to_string()) + } + } else { + Ok(date.to_string()) + } + } + fn in_date_range_extended(&self, member_sql: &str) -> Result { + let (from, to) = self.allocate_date_params()?; + + let from = if self.values.len() >= 3 { + self.extend_date_range_bound(from, &self.values[2], true)? + } else { + from + }; + + let to = if self.values.len() >= 4 { + self.extend_date_range_bound(to, &self.values[3], false)? + } else { + to + }; + + self.templates + .time_range_filter(member_sql.to_string(), from, to) + } + fn in_where(&self, member_sql: &str) -> Result { let need_null_check = self.is_need_null_chek(false); self.templates.in_where( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs index d103229021eeb..17e8a64a6be20 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs @@ -1,4 +1,5 @@ use super::base_filter::{BaseFilter, FilterType}; +use super::FilterOperator; use crate::cube_bridge::base_query_options::FilterItem as NativeFilterItem; use crate::plan::filter::{FilterGroup, FilterGroupOperator, FilterItem}; use crate::planner::query_tools::QueryTools; @@ -6,6 +7,7 @@ use crate::planner::sql_evaluator::Compiler; use crate::planner::BaseTimeDimension; use cubenativeutils::CubeError; use std::rc::Rc; +use std::str::FromStr; pub struct FilterCompiler<'a> { evaluator_compiler: &'a mut Compiler, @@ -43,7 +45,7 @@ impl<'a> FilterCompiler<'a> { self.query_tools.clone(), item.member_evaluator(), FilterType::Dimension, - "InDateRange".to_string(), + FilterOperator::InDateRange, Some(date_range.into_iter().map(|v| Some(v)).collect()), )?; self.time_dimension_filters.push(FilterItem::Item(filter)); @@ -92,7 +94,7 @@ impl<'a> FilterCompiler<'a> { self.query_tools.clone(), evaluator, item_type.clone(), - operator.clone(), + FilterOperator::from_str(&operator)?, item.values.clone(), )?)) } else { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs index 5b3eeae125f06..d66caf4368928 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs @@ -1,9 +1,12 @@ use cubenativeutils::CubeError; use std::str::FromStr; + +#[derive(Clone, PartialEq, Debug)] pub enum FilterOperator { Equal, NotEqual, InDateRange, + InDateRangeExtended, In, NotIn, Set, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/mod.rs index 02277136c6979..01f95fd0d8ba6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/mod.rs @@ -3,3 +3,4 @@ pub mod compiler; pub mod filter_operator; pub use base_filter::BaseFilter; +pub use filter_operator::FilterOperator; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/granularity_helper.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/granularity_helper.rs new file mode 100644 index 0000000000000..b1e4bf1863b80 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/granularity_helper.rs @@ -0,0 +1,149 @@ +use cubenativeutils::CubeError; +use itertools::Itertools; +use lazy_static::lazy_static; +use std::collections::HashMap; + +pub struct GranularityHelper {} + +impl GranularityHelper { + pub fn min_granularity( + granularity_a: &Option, + granularity_b: &Option, + ) -> Result, CubeError> { + if let Some((granularity_a, granularity_b)) = + granularity_a.as_ref().zip(granularity_b.as_ref()) + { + let a_parents = Self::granularity_parents(granularity_a)?; + let b_parents = Self::granularity_parents(granularity_b)?; + let diff_position = a_parents + .iter() + .zip(b_parents.iter()) + .find_position(|(a, b)| a != b); + if let Some((diff_position, _)) = diff_position { + if diff_position == 0 { + Err(CubeError::user(format!( + "Can't find common parent for '{granularity_a}' and '{granularity_b}'" + ))) + } else { + Ok(Some(a_parents[diff_position - 1].clone())) + } + } else { + if a_parents.len() >= b_parents.len() { + Ok(Some(b_parents.last().unwrap().clone())) + } else { + Ok(Some(a_parents.last().unwrap().clone())) + } + } + } else if granularity_a.is_some() { + Ok(granularity_a.clone()) + } else { + Ok(granularity_b.clone()) + } + } + + pub fn granularity_from_interval(interval: &Option) -> Option { + if let Some(interval) = interval { + if interval.find("day").is_some() { + Some("day".to_string()) + } else if interval.find("month").is_some() { + Some("month".to_string()) + } else if interval.find("year").is_some() { + Some("year".to_string()) + } else if interval.find("week").is_some() { + Some("week".to_string()) + } else if interval.find("hour").is_some() { + Some("hour".to_string()) + } else { + None + } + } else { + None + } + } + + pub fn granularity_parents(granularity: &str) -> Result<&Vec, CubeError> { + if let Some(parents) = Self::standard_granularity_parents().get(granularity) { + Ok(parents) + } else { + Err(CubeError::user(format!( + "Granularity {} not found", + granularity + ))) + } + } + + pub fn standard_granularity_parents() -> &'static HashMap> { + lazy_static! { + static ref STANDARD_GRANULARITIES_PARENTS: HashMap> = { + let mut map = HashMap::new(); + map.insert( + "year".to_string(), + vec![ + "second".to_string(), + "minute".to_string(), + "hour".to_string(), + "day".to_string(), + "month".to_string(), + "quarter".to_string(), + "year".to_string(), + ], + ); + map.insert( + "quarter".to_string(), + vec![ + "second".to_string(), + "minute".to_string(), + "hour".to_string(), + "day".to_string(), + "month".to_string(), + "quarter".to_string(), + ], + ); + map.insert( + "month".to_string(), + vec![ + "second".to_string(), + "minute".to_string(), + "hour".to_string(), + "day".to_string(), + "month".to_string(), + ], + ); + map.insert( + "week".to_string(), + vec![ + "second".to_string(), + "minute".to_string(), + "hour".to_string(), + "day".to_string(), + "week".to_string(), + ], + ); + map.insert( + "day".to_string(), + vec![ + "second".to_string(), + "minute".to_string(), + "hour".to_string(), + "day".to_string(), + ], + ); + map.insert( + "hour".to_string(), + vec![ + "second".to_string(), + "minute".to_string(), + "hour".to_string(), + ], + ); + map.insert( + "minute".to_string(), + vec!["second".to_string(), "minute".to_string()], + ); + map.insert("second".to_string(), vec!["second".to_string()]); + map + }; + } + &STANDARD_GRANULARITIES_PARENTS + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/mod.rs index a6def37da64c1..cc457b6318ecc 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/mod.rs @@ -6,6 +6,7 @@ pub mod base_member; pub mod base_query; pub mod base_time_dimension; pub mod filter; +pub mod granularity_helper; pub mod params_allocator; pub mod planners; pub mod query_properties; @@ -22,6 +23,7 @@ pub use base_measure::BaseMeasure; pub use base_member::{BaseMember, BaseMemberHelper}; pub use base_query::BaseQuery; pub use base_time_dimension::BaseTimeDimension; +pub use granularity_helper::GranularityHelper; pub use params_allocator::ParamsAllocator; pub use query_properties::{FullKeyAggregateMeasures, OrderByItem, QueryProperties}; pub use visitor_context::{evaluate_with_context, VisitorContext}; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs index c47854234ad8d..5f729bb51f8ad 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs @@ -1,5 +1,7 @@ +use crate::plan::{FilterGroup, FilterItem}; +use crate::planner::filter::FilterOperator; use crate::planner::planners::multi_stage::MultiStageTimeShift; -use crate::planner::BaseDimension; +use crate::planner::{BaseDimension, BaseTimeDimension}; use itertools::Itertools; use std::cmp::PartialEq; use std::collections::{HashMap, HashSet}; @@ -8,27 +10,39 @@ use std::rc::Rc; #[derive(Clone)] pub struct MultiStageAppliedState { + time_dimensions: Vec>, dimensions: Vec>, - allowed_filter_members: HashSet, + time_dimensions_filters: Vec, + dimensions_filters: Vec, + measures_filters: Vec, time_shifts: HashMap, } impl MultiStageAppliedState { pub fn new( + time_dimensions: Vec>, dimensions: Vec>, - allowed_filter_members: HashSet, + time_dimensions_filters: Vec, + dimensions_filters: Vec, + measures_filters: Vec, ) -> Rc { Rc::new(Self { + time_dimensions, dimensions, - allowed_filter_members, + time_dimensions_filters, + dimensions_filters, + measures_filters, time_shifts: HashMap::new(), }) } pub fn clone_state(&self) -> Self { Self { + time_dimensions: self.time_dimensions.clone(), dimensions: self.dimensions.clone(), - allowed_filter_members: self.allowed_filter_members.clone(), + time_dimensions_filters: self.time_dimensions_filters.clone(), + dimensions_filters: self.dimensions_filters.clone(), + measures_filters: self.measures_filters.clone(), time_shifts: self.time_shifts.clone(), } } @@ -54,35 +68,165 @@ impl MultiStageAppliedState { &self.time_shifts } - pub fn is_filter_allowed(&self, name: &str) -> bool { - self.allowed_filter_members.contains(name) + pub fn time_dimensions_filters(&self) -> &Vec { + &self.time_dimensions_filters } - pub fn allowed_filter_members(&self) -> &HashSet { - &self.allowed_filter_members + pub fn dimensions_filters(&self) -> &Vec { + &self.dimensions_filters } - pub fn disallow_filter(&mut self, name: &str) { - self.allowed_filter_members.take(name); + pub fn measures_filters(&self) -> &Vec { + &self.measures_filters } pub fn dimensions(&self) -> &Vec> { &self.dimensions } + + pub fn time_dimensions(&self) -> &Vec> { + &self.time_dimensions + } + + pub fn change_time_dimension_granularity( + &mut self, + dimension_name: &str, + new_granularity: Option, + ) { + if let Some(time_dimension) = self + .time_dimensions + .iter_mut() + .find(|dim| dim.member_evaluator().full_name() == dimension_name) + { + *time_dimension = time_dimension.change_granularity(new_granularity); + } + } + + pub fn remove_filter_for_member(&mut self, member_name: &String) { + self.time_dimensions_filters = + self.extract_filters_exclude_member(member_name, &self.time_dimensions_filters); + self.dimensions_filters = + self.extract_filters_exclude_member(member_name, &self.dimensions_filters); + self.measures_filters = + self.extract_filters_exclude_member(member_name, &self.measures_filters); + } + + fn extract_filters_exclude_member( + &self, + member_name: &String, + filters: &Vec, + ) -> Vec { + let mut result = Vec::new(); + for item in filters.iter() { + match item { + FilterItem::Group(group) => { + let new_group = FilterItem::Group(Rc::new(FilterGroup::new( + group.operator.clone(), + self.extract_filters_exclude_member(member_name, &group.items), + ))); + result.push(new_group); + } + FilterItem::Item(itm) => { + if &itm.member_name() != member_name { + result.push(FilterItem::Item(itm.clone())); + } + } + } + } + result + } + + pub fn has_filters_for_member(&self, member_name: &String) -> bool { + self.has_filters_for_member_impl(member_name, &self.time_dimensions_filters) + || self.has_filters_for_member_impl(member_name, &self.dimensions_filters) + || self.has_filters_for_member_impl(member_name, &self.measures_filters) + } + + fn has_filters_for_member_impl(&self, member_name: &String, filters: &Vec) -> bool { + for item in filters.iter() { + match item { + FilterItem::Group(group) => { + if self.has_filters_for_member_impl(member_name, &group.items) { + return true; + } + } + FilterItem::Item(itm) => { + if &itm.member_name() == member_name { + return true; + } + } + } + } + false + } + + pub fn expand_date_range_filter( + &mut self, + member_name: &String, + left_interval: Option, + right_interval: Option, + ) { + self.time_dimensions_filters = self.expand_date_range_filter_impl( + member_name, + &self.time_dimensions_filters, + &left_interval, + &right_interval, + ); + } + + fn expand_date_range_filter_impl( + &self, + member_name: &String, + filters: &Vec, + left_interval: &Option, + right_interval: &Option, + ) -> Vec { + let mut result = Vec::new(); + for item in filters.iter() { + match item { + FilterItem::Group(group) => { + let new_group = FilterItem::Group(Rc::new(FilterGroup::new( + group.operator.clone(), + self.expand_date_range_filter_impl( + member_name, + filters, + left_interval, + right_interval, + ), + ))); + result.push(new_group); + } + FilterItem::Item(itm) => { + let itm = if &itm.member_name() == member_name + && matches!(itm.filter_operator(), FilterOperator::InDateRange) + { + let mut values = itm.values().clone(); + values.push(left_interval.clone()); + values.push(right_interval.clone()); + itm.change_operator(FilterOperator::InDateRangeExtended, values) + } else { + itm.clone() + }; + result.push(FilterItem::Item(itm)); + } + } + } + result + } } impl PartialEq for MultiStageAppliedState { fn eq(&self, other: &Self) -> bool { - let dims_eq = if !self.dimensions.len() == other.dimensions.len() { - false - } else { - self.dimensions + let dims_eq = self.dimensions.len() == other.dimensions.len() + && self + .dimensions .iter() .zip(other.dimensions.iter()) - .all(|(a, b)| a.member_evaluator().full_name() == b.member_evaluator().full_name()) - }; + .all(|(a, b)| a.member_evaluator().full_name() == b.member_evaluator().full_name()); dims_eq - && self.allowed_filter_members == other.allowed_filter_members + && self.time_dimensions_filters == other.time_dimensions_filters + && self.dimensions_filters == other.dimensions_filters + && self.measures_filters == other.measures_filters && self.time_shifts == other.time_shifts } } @@ -98,7 +242,6 @@ impl Debug for MultiStageAppliedState { .map(|d| d.member_evaluator().full_name()) .join(", "), ) - .field("allowed_filter_members", &self.allowed_filter_members) .field("time_shifts", &self.time_shifts) .finish() } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs index 6bb827a7c8c2e..34c7b059c4252 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs @@ -73,12 +73,18 @@ pub struct RollingWindowDescription { pub offset: String, } +#[derive(Clone)] +pub struct RunningTotalDescription { + pub time_dimension: Rc, +} + #[derive(Clone)] pub enum MultiStageInodeMemberType { Rank, Aggregate, Calculate, RollingWindow(RollingWindowDescription), + RunningTotal(RunningTotalDescription), } #[derive(Clone)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index a2d73ee0bda9e..8d681c3dbc6e8 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -109,7 +109,7 @@ impl MultiStageMemberQueryPlanner { rolling_window_desc.time_dimension.clone(), ); let cte_schema = cte_schemas.get(input).unwrap().clone(); - join_builder.inner_join_table_reference( + join_builder.left_join_table_reference( input.clone(), cte_schema, Some(format!("rolling_{}", i + 1)), @@ -125,11 +125,24 @@ impl MultiStageMemberQueryPlanner { .collect_vec(); let context_factory = SqlNodesFactory::new(); - let node_context = context_factory.default_node_processor(); + let node_context = context_factory.rolling_window_node_processor(); let mut select_builder = SelectBuilder::new(from, VisitorContext::new(None, node_context)); for dim in dimensions.iter() { - select_builder.add_projection_member(&dim, None, None); + if dim.full_name() == rolling_window_desc.time_dimension.full_name() { + select_builder.add_projection_member( + &dim, + Some(root_alias.clone()), + Some( + cte_schemas + .get(&inputs[1]) + .unwrap() + .resolve_member_alias(&dim, &Some(inputs[1].clone())), + ), + ); + } else { + select_builder.add_projection_member(&dim, None, None); + } } let query_member = self.query_member_as_base_member()?; @@ -267,29 +280,17 @@ impl MultiStageMemberQueryPlanner { { vec![measure] } else { - println!("!!! kkkk {}", self.description.alias()); vec![] }; - let allowed_filter_members = self.description.state().allowed_filter_members().clone(); - let cte_query_properties = QueryProperties::try_new_from_precompiled( self.query_tools.clone(), measures, self.description.state().dimensions().clone(), - self.query_properties.time_dimensions().clone(), - self.extract_filters( - &allowed_filter_members, - self.query_properties.time_dimensions_filters(), - ), - self.extract_filters( - &allowed_filter_members, - self.query_properties.dimensions_filters(), - ), - self.extract_filters( - &allowed_filter_members, - self.query_properties.measures_filters(), - ), + self.description.state().time_dimensions().clone(), + self.description.state().time_dimensions_filters().clone(), + self.description.state().dimensions_filters().clone(), + self.description.state().measures_filters().clone(), vec![], None, None, @@ -364,7 +365,7 @@ impl MultiStageMemberQueryPlanner { fn all_dimensions(&self) -> Vec> { BaseMemberHelper::iter_as_base_member(self.description.state().dimensions()) .chain(BaseMemberHelper::iter_as_base_member( - self.query_properties.time_dimensions(), + self.description.state().time_dimensions(), )) .collect_vec() } @@ -381,7 +382,7 @@ impl MultiStageMemberQueryPlanner { fn all_input_dimensions(&self) -> Vec> { BaseMemberHelper::iter_as_base_member(&self.input_dimensions()) .chain(BaseMemberHelper::iter_as_base_member( - self.query_properties.time_dimensions(), + self.description.state().time_dimensions(), )) .collect_vec() } @@ -488,7 +489,7 @@ impl MultiStageMemberQueryPlanner { fn subquery_order(&self) -> Result, CubeError> { let order_items = QueryProperties::default_order( &self.input_dimensions(), - &self.query_properties.time_dimensions(), + &self.description.state().time_dimensions(), &self.raw_input_measures()?, ); Ok(OrderPlanner::custom_order( @@ -506,7 +507,7 @@ impl MultiStageMemberQueryPlanner { let order_items = QueryProperties::default_order( &self.description.state().dimensions(), - &self.query_properties.time_dimensions(), + &self.description.state().time_dimensions(), &measures, ); let mut all_members = self.all_dimensions().clone(); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs index d5b8e965d19b7..1237a0ece2c75 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs @@ -3,6 +3,7 @@ use super::multi_stage::{ MultiStageLeafMemberType, MultiStageMember, MultiStageMemberQueryPlanner, MultiStageMemberType, MultiStageQueryDescription, MultiStageTimeShift, RollingWindowDescription, }; +use crate::cube_bridge::measure_definition::RollingWindow; use crate::plan::{Cte, From, Schema, Select, SelectBuilder}; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::collectors::has_multi_stage_members; @@ -10,7 +11,7 @@ use crate::planner::sql_evaluator::collectors::member_childs; use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory; use crate::planner::sql_evaluator::EvaluationNode; use crate::planner::{BaseDimension, BaseMeasure, VisitorContext}; -use crate::planner::{BaseTimeDimension, QueryProperties}; +use crate::planner::{BaseTimeDimension, GranularityHelper, QueryProperties}; use cubenativeutils::CubeError; use itertools::Itertools; use std::collections::HashMap; @@ -45,10 +46,12 @@ impl MultiStageQueryPlanner { return Ok((vec![], vec![])); } let mut descriptions = Vec::new(); - let all_filter_members = self.query_properties.all_filtered_members(); let state = MultiStageAppliedState::new( + self.query_properties.time_dimensions().clone(), self.query_properties.dimensions().clone(), - all_filter_members, + self.query_properties.time_dimensions_filters().clone(), + self.query_properties.dimensions_filters().clone(), + self.query_properties.measures_filters().clone(), ); let top_level_ctes = multi_stage_members @@ -190,7 +193,7 @@ impl MultiStageQueryPlanner { MultiStageMemberType::Leaf(MultiStageLeafMemberType::Measure), member, ), - state.clone(), + state, vec![], alias.clone(), ); @@ -198,6 +201,36 @@ impl MultiStageQueryPlanner { Ok(description) } + fn make_rolling_base_state( + &self, + time_dimension: Rc, + rolling_window: &RollingWindow, + state: Rc, + ) -> Result, CubeError> { + let time_dimension_name = time_dimension.member_evaluator().full_name(); + let mut new_state = state.clone_state(); + let trailing_granularity = + GranularityHelper::granularity_from_interval(&rolling_window.trailing); + let leading_granularity = + GranularityHelper::granularity_from_interval(&rolling_window.leading); + let window_granularity = + GranularityHelper::min_granularity(&trailing_granularity, &leading_granularity)?; + let result_granularity = GranularityHelper::min_granularity( + &window_granularity, + &time_dimension.get_granularity(), + )?; + + new_state.change_time_dimension_granularity(&time_dimension_name, result_granularity); + + new_state.expand_date_range_filter( + &time_dimension_name, + rolling_window.trailing.clone(), + rolling_window.leading.clone(), + ); + + Ok(Rc::new(new_state)) + } + fn try_make_rolling_window( &self, member: Rc, @@ -205,8 +238,22 @@ impl MultiStageQueryPlanner { descriptions: &mut Vec>, ) -> Result>, CubeError> { if let Some(measure) = BaseMeasure::try_new(member.clone(), self.query_tools.clone())? { - if let Some(rolling_window) = measure.rolling_window() { + if measure.is_cumulative() { + let rolling_window = if let Some(rolling_window) = measure.rolling_window() { + rolling_window.clone() + } else { + RollingWindow { + trailing: Some("unbounded".to_string()), + leading: None, + offset: None, + } + }; let time_dimensions = self.query_properties.time_dimensions(); + if time_dimensions.len() == 0 { + let rolling_base = + self.add_rolling_window_base(member.clone(), state.clone(), descriptions)?; + return Ok(Some(rolling_base)); + } if time_dimensions.len() != 1 { return Err(CubeError::internal( "Rolling window requires one time dimension".to_string(), @@ -219,7 +266,15 @@ impl MultiStageQueryPlanner { let input = vec![ self.add_time_series(time_dimension.clone(), state.clone(), descriptions)?, - self.add_rolling_window_base(source.clone(), state.clone(), descriptions)?, + self.add_rolling_window_base( + source.clone(), + self.make_rolling_base_state( + time_dimension.clone(), + &rolling_window, + state.clone(), + )?, + descriptions, + )?, ]; let time_dimension = time_dimensions[0].clone(); @@ -308,7 +363,7 @@ impl MultiStageQueryPlanner { let new_state = if !dimensions_to_add.is_empty() || !multi_stage_member.time_shifts().is_empty() - || state.is_filter_allowed(&member_name) + || state.has_filters_for_member(&member_name) { let mut new_state = state.clone_state(); if !dimensions_to_add.is_empty() { @@ -317,8 +372,8 @@ impl MultiStageQueryPlanner { if !multi_stage_member.time_shifts().is_empty() { new_state.add_time_shifts(multi_stage_member.time_shifts().clone()); } - if state.is_filter_allowed(&member_name) { - new_state.disallow_filter(&member_name); + if state.has_filters_for_member(&member_name) { + new_state.remove_filter_for_member(&member_name); } Rc::new(new_state) } else { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs index 30ed48ab13893..f7b8b6234be21 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs @@ -26,7 +26,9 @@ impl TraversalVisitor for HasMultiStageMembersCollector { MemberSymbolType::Measure(s) => { if s.is_multi_stage() { self.has_multi_stage = true; - } else if !self.ignore_cumulative && s.is_rolling_window() { + } else if !self.ignore_cumulative + && (s.is_rolling_window() || s.measure_type() == "runningTotal") + { self.has_multi_stage = true; } else { for filter_node in s.measure_filters() { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs index d7e580edcac19..a8a63d4f21b55 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs @@ -2,7 +2,8 @@ use crate::plan::schema::Schema; use crate::planner::sql_evaluator::sql_nodes::final_measure::FinalMeasureSqlNode; use crate::planner::sql_evaluator::sql_nodes::{ AutoPrefixSqlNode, EvaluateSqlNode, MeasureFilterSqlNode, MultiStageRankNode, - MultiStageWindowNode, RenderReferencesSqlNode, RootSqlNode, SqlNode, TimeShiftSqlNode, + MultiStageWindowNode, RenderReferencesSqlNode, RollingWindowNode, RootSqlNode, SqlNode, + TimeShiftSqlNode, }; use std::rc::Rc; @@ -55,6 +56,13 @@ pub fn set_schema_impl(sql_node: Rc, schema: Rc) -> Rc() + { + let input = set_schema_impl(rolling_window.input().clone(), schema.clone()); + RollingWindowNode::new(input) } else if let Some(render_references) = sql_node .clone() .as_any() @@ -81,6 +89,6 @@ pub fn set_schema_impl(sql_node: Rc, schema: Rc) -> Rc ev.evaluate_sql(args), - MemberSymbolType::Measure(ev) => { - let res = if ev.is_splitted_source() { - //FIXME hack for working with - //measures like rolling window - if !args.is_empty() { - match &args[0] { - MemberSqlArg::String(s) => s.clone(), - _ => "".to_string(), - } - } else { - "".to_string() - } - } else { - ev.evaluate_sql(args)? - }; - Ok(res) - } + MemberSymbolType::Measure(ev) => ev.evaluate_sql(args), MemberSymbolType::CubeTable(ev) => ev.evaluate_sql(args), MemberSymbolType::CubeName(ev) => ev.evaluate_sql(args), MemberSymbolType::SimpleSql(ev) => ev.evaluate_sql(args), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs index 9d8729896e84d..3368a23781deb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs @@ -1,7 +1,7 @@ use super::{ AutoPrefixSqlNode, EvaluateSqlNode, FinalMeasureSqlNode, MeasureFilterSqlNode, - MultiStageRankNode, MultiStageWindowNode, RenderReferencesSqlNode, RootSqlNode, SqlNode, - TimeShiftSqlNode, + MultiStageRankNode, MultiStageWindowNode, RenderReferencesSqlNode, RollingWindowNode, + RootSqlNode, SqlNode, TimeShiftSqlNode, }; use std::collections::HashMap; use std::rc::Rc; @@ -73,6 +73,24 @@ impl SqlNodesFactory { references_processor } + pub fn rolling_window_node_processor(&self) -> Rc { + let evaluate_sql_processor = EvaluateSqlNode::new(); + let auto_prefix_processor = AutoPrefixSqlNode::new(evaluate_sql_processor.clone()); + let measure_filter_processor = MeasureFilterSqlNode::new(auto_prefix_processor.clone()); + let final_measure_processor = FinalMeasureSqlNode::new(measure_filter_processor.clone()); + + let rolling_window_processor = RollingWindowNode::new(final_measure_processor.clone()); + + let root_processor = RootSqlNode::new( + self.dimension_processor(auto_prefix_processor.clone()), + rolling_window_processor.clone(), + auto_prefix_processor.clone(), + evaluate_sql_processor.clone(), + ); + let references_processor = RenderReferencesSqlNode::new(root_processor); + references_processor + } + fn dimension_processor(&self, input: Rc) -> Rc { if let Some(time_shifts) = &self.time_shifts { TimeShiftSqlNode::new(time_shifts.clone(), input) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs index 2fd786a986f9e..c5469d5206242 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs @@ -40,7 +40,11 @@ impl SqlNode for FinalMeasureSqlNode { if ev.is_calculated() { input } else { - let measure_type = ev.measure_type(); + let measure_type = if ev.measure_type() == "runningTotal" { + "sum" + } else { + &ev.measure_type() + }; format!("{}({})", measure_type, input) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs index 9cb6190581a9d..f316c5d8facb3 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs @@ -6,6 +6,7 @@ pub mod measure_filter; pub mod multi_stage_rank; pub mod multi_stage_window; pub mod render_references; +pub mod rolling_window; pub mod root_processor; pub mod sql_node; pub mod time_shift; @@ -18,6 +19,7 @@ pub use measure_filter::MeasureFilterSqlNode; pub use multi_stage_rank::MultiStageRankNode; pub use multi_stage_window::MultiStageWindowNode; pub use render_references::RenderReferencesSqlNode; +pub use rolling_window::RollingWindowNode; pub use root_processor::RootSqlNode; pub use sql_node::SqlNode; pub use time_shift::TimeShiftSqlNode; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs new file mode 100644 index 0000000000000..f97ac113e2fc4 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs @@ -0,0 +1,76 @@ +use super::SqlNode; +use crate::cube_bridge::memeber_sql::MemberSqlArg; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType, SqlEvaluatorVisitor}; +use cubenativeutils::CubeError; +use std::any::Any; +use std::rc::Rc; + +pub struct RollingWindowNode { + input: Rc, +} + +impl RollingWindowNode { + pub fn new(input: Rc) -> Rc { + Rc::new(Self { input }) + } + + pub fn input(&self) -> &Rc { + &self.input + } +} + +impl SqlNode for RollingWindowNode { + fn to_sql( + &self, + visitor: &mut SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, + node_processor: Rc, + ) -> Result { + let res = match node.symbol() { + MemberSymbolType::Measure(m) => { + if m.is_cumulative() && m.is_splitted_source() { + let args = visitor.evaluate_deps(node, node_processor.clone())?; + //FIXME hack for working with + //measures like rolling window + let input = if !args.is_empty() { + match &args[0] { + MemberSqlArg::String(s) => s.clone(), + _ => "".to_string(), + } + } else { + "".to_string() + }; + let aggregate_function = if m.measure_type() == "sum" + || m.measure_type() == "count" + || m.measure_type() == "runningTotal" + { + "sum" + } else { + m.measure_type() + }; + + format!("{}({})", aggregate_function, input) + } else { + self.input + .to_sql(visitor, node, query_tools.clone(), node_processor.clone())? + } + } + _ => { + return Err(CubeError::internal(format!( + "Unexpected evaluation node type for RollingWindowNode" + ))); + } + }; + Ok(res) + } + + fn as_any(self: Rc) -> Rc { + self.clone() + } + + fn childs(&self) -> Vec> { + vec![self.input.clone()] + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs index be6d0b60603c6..261ec2c5f7e8b 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs @@ -115,6 +115,14 @@ impl MeasureSymbol { self.rolling_window().is_some() } + pub fn is_running_total(&self) -> bool { + self.measure_type() == "runningTotal" + } + + pub fn is_cumulative(&self) -> bool { + self.is_rolling_window() || self.is_running_total() + } + pub fn measure_filters(&self) -> &Vec> { &self.measure_filters } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs index f0d23de97e232..c7eaaf9a8ed11 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs @@ -3,6 +3,7 @@ use cubenativeutils::CubeError; use minijinja::context; use std::rc::Rc; +#[derive(Clone)] pub struct FilterTemplates { render: Rc, } @@ -103,6 +104,26 @@ impl FilterTemplates { ) } + pub fn add_interval(&self, date: String, interval: String) -> Result { + self.render.render_template( + &"expressions/add_interval", + context! { + date => date, + interval => interval + }, + ) + } + + pub fn sub_interval(&self, date: String, interval: String) -> Result { + self.render.render_template( + &"expressions/sub_interval", + context! { + date => date, + interval => interval + }, + ) + } + pub fn set_where(&self, column: String) -> Result { self.render.render_template( &"filters/set_where", From ef08b6a1776928dfadfc17ac04fb58321687f194 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Wed, 4 Dec 2024 10:27:42 +0100 Subject: [PATCH 5/6] update --- .../src/adapter/BaseQuery.js | 31 +++---------------- .../postgres/sql-generation.test.ts | 3 +- 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index d325eb455cc73..5d6e3fa86cee5 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -11,7 +11,7 @@ import cronParser from 'cron-parser'; import moment from 'moment-timezone'; import inflection from 'inflection'; -import { FROM_PARTITION_RANGE, inDbTimeZone, MAX_SOURCE_ROW_LIMIT, QueryAlias, getEnv, timeSeries } from '@cubejs-backend/shared'; +import { FROM_PARTITION_RANGE, inDbTimeZone, MAX_SOURCE_ROW_LIMIT, QueryAlias, getEnv, timeSeries as timeSeriesBase } from '@cubejs-backend/shared'; import { buildSqlAndParams as nativeBuildSqlAndParams, @@ -576,29 +576,6 @@ export class BaseQuery { return false; } - buildSqlAndParamsTest(exportAnnotatedSql) { - if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) { - if (this.externalPreAggregationQuery()) { // TODO performance - return this.externalQuery().buildSqlAndParams(exportAnnotatedSql); - } - } - const js_res = this.compilers.compiler.withQuery( - this, - () => this.cacheValue( - ['buildSqlAndParams', exportAnnotatedSql], - () => this.paramAllocator.buildSqlAndParams( - this.buildParamAnnotatedSql(), - exportAnnotatedSql, - this.shouldReuseParams - ), - { cache: this.queryCache } - ) - ); - const rust = this.buildSqlAndParamsRust(exportAnnotatedSql); - console.log('js result: ', js_res[0]); - console.log('rust result: ', rust[0]); - return js_res; - } /** * Returns a pair of SQL query string and parameter values for the query. * @param {boolean} [exportAnnotatedSql] - returns annotated sql with not rendered params if true @@ -651,10 +628,11 @@ export class BaseQuery { return res; } - //FIXME helper for native generator, maybe should be moved entire to rust + // FIXME helper for native generator, maybe should be moved entire to rust generateTimeSeries(granularity, dateRange) { - return timeSeries(granularity, dateRange); + return timeSeriesBase(granularity, dateRange); } + get shouldReuseParams() { return false; } @@ -1419,7 +1397,6 @@ export class BaseQuery { ) ).join(' AND '); - return this.overTimeSeriesSelect( cumulativeMeasures, dateSeriesSql, diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index 103a9f6dbee12..a595e0523dc17 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -666,7 +666,7 @@ describe('SQL Generation', () => { console.log(query.buildSqlAndParams()); // TODO ordering doesn't work for running total - return dbRunner.testQuery(query.buildSqlAndParamsTest()).then(res => { + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { console.log(JSON.stringify(res)); expect(res).toEqual( [{ @@ -916,7 +916,6 @@ describe('SQL Generation', () => { { visitors__created_at_day: '2017-01-10T00:00:00.000Z', visitors__running_revenue_per_count: '300' } ])); - it('hll rolling (BigQuery)', async () => { await compiler.compile(); From 1d4f34f29a16c4425b074a56b8c93b7936aee3a4 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Wed, 4 Dec 2024 10:55:18 +0100 Subject: [PATCH 6/6] update --- .../cubesqlplanner/src/plan/join.rs | 2 +- .../cubesqlplanner/src/plan/time_series.rs | 3 +- .../src/planner/filter/base_filter.rs | 1 - .../planners/multi_stage/applied_state.rs | 2 +- .../planner/planners/multi_stage/member.rs | 2 +- .../multi_stage/member_query_planner.rs | 33 +++---------------- .../sql_evaluator/sql_nodes/evaluate_sql.rs | 1 - .../sql_evaluator/symbols/member_symbol.rs | 2 -- 8 files changed, 8 insertions(+), 38 deletions(-) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs index bd32f03c2b825..eb3a7347bd103 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs @@ -1,4 +1,4 @@ -use super::{time_series, Schema, SingleAliasedSource}; +use super::{Schema, SingleAliasedSource}; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::{BaseJoinCondition, BaseMember, VisitorContext}; use cubenativeutils::CubeError; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs index 774fa0e825878..f9e0ce8f777d5 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs @@ -1,7 +1,6 @@ -use super::{Schema, SchemaColumn, Select, Union}; +use super::{Schema, SchemaColumn}; use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; -use std::rc::Rc; pub struct TimeSeries { pub time_dimension_name: String, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs index fb1de9080e192..249092a5774f2 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs @@ -8,7 +8,6 @@ use cubenativeutils::CubeError; use lazy_static::lazy_static; use regex::Regex; use std::rc::Rc; -use std::str::FromStr; #[derive(Debug, Clone, PartialEq, Eq)] pub enum FilterType { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs index 5f729bb51f8ad..7a4061c43f64d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs @@ -4,7 +4,7 @@ use crate::planner::planners::multi_stage::MultiStageTimeShift; use crate::planner::{BaseDimension, BaseTimeDimension}; use itertools::Itertools; use std::cmp::PartialEq; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::fmt::Debug; use std::rc::Rc; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs index 34c7b059c4252..dbfa67cc2f7ba 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs @@ -1,4 +1,4 @@ -use crate::cube_bridge::measure_definition::{MeasureDefinition, TimeShiftReference}; +use crate::cube_bridge::measure_definition::TimeShiftReference; use crate::planner::sql_evaluator::EvaluationNode; use crate::planner::BaseMember; use crate::planner::BaseTimeDimension; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index 8d681c3dbc6e8..06533fcbed1f2 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -21,7 +21,7 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; pub struct MultiStageMemberQueryPlanner { query_tools: Rc, - query_properties: Rc, + _query_properties: Rc, description: Rc, } @@ -33,7 +33,7 @@ impl MultiStageMemberQueryPlanner { ) -> Self { Self { query_tools, - query_properties, + _query_properties: query_properties, description, } } @@ -83,7 +83,7 @@ impl MultiStageMemberQueryPlanner { fn plan_rolling_window_query( &self, rolling_window_desc: &RollingWindowDescription, - multi_stage_member: &MultiStageInodeMember, + _multi_stage_member: &MultiStageInodeMember, cte_schemas: &HashMap>, ) -> Result, CubeError> { let inputs = self.input_cte_aliases(); @@ -226,7 +226,7 @@ impl MultiStageMemberQueryPlanner { fn make_input_join( &self, - multi_stage_member: &MultiStageInodeMember, + _multi_stage_member: &MultiStageInodeMember, cte_schemas: &HashMap>, ) -> Result { let inputs = self.input_cte_aliases(); @@ -337,31 +337,6 @@ impl MultiStageMemberQueryPlanner { Ok(Rc::new(result)) } - fn extract_filters( - &self, - allowed_filter_members: &HashSet, - filters: &Vec, - ) -> Vec { - let mut result = Vec::new(); - for item in filters.iter() { - match item { - FilterItem::Group(group) => { - let new_group = FilterItem::Group(Rc::new(FilterGroup::new( - group.operator.clone(), - self.extract_filters(allowed_filter_members, &group.items), - ))); - result.push(new_group); - } - FilterItem::Item(itm) => { - if allowed_filter_members.contains(&itm.member_name()) { - result.push(FilterItem::Item(itm.clone())); - } - } - } - } - result - } - fn all_dimensions(&self) -> Vec> { BaseMemberHelper::iter_as_base_member(self.description.state().dimensions()) .chain(BaseMemberHelper::iter_as_base_member( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs index 2550fda974ee4..0acd90c5f868d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs @@ -1,5 +1,4 @@ use super::SqlNode; -use crate::cube_bridge::memeber_sql::MemberSqlArg; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs index d8d3ce1836ce1..b8137abda10cb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs @@ -1,5 +1,3 @@ -use std::rc::Rc; - pub trait MemberSymbol { fn cube_name(&self) -> &String; fn name(&self) -> &String;