diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 5d6e3fa86cee5..8ee072bafbd5a 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -1036,8 +1036,10 @@ export class BaseQuery { R.uniq, R.map(m => this.newMeasure(m)) ); + console.log("!!!! meas hierarchy", measureToHierarchy); const multipliedMeasures = measuresToRender(true, false)(measureToHierarchy); + console.log("!!! mul meas", multipliedMeasures); const regularMeasures = measuresToRender(false, false)(measureToHierarchy); const cumulativeMeasures = @@ -1775,6 +1777,7 @@ export class BaseQuery { return measures.map(measure => { const cubes = this.collectFrom([measure], this.collectCubeNamesFor.bind(this), 'collectCubeNamesFor'); const joinHints = this.collectFrom([measure], this.collectJoinHintsFor.bind(this), 'collectJoinHintsFor'); + console.log("!!! cubes: ", cubes, " ", joinHints); if (R.any(cubeName => keyCubeName !== cubeName, cubes)) { const measuresJoin = this.joinGraph.buildJoin(joinHints); if (measuresJoin.multiplicationFactor[keyCubeName]) { @@ -2481,6 +2484,7 @@ export class BaseQuery { fn, renderContext ); + console.log("!!!!! renderContext.measuresToRender.length ", renderContext.measuresToRender.length); return renderContext.measuresToRender.length ? R.uniq(renderContext.measuresToRender) : [renderContext.rootMeasure.value]; @@ -3288,6 +3292,7 @@ export class BaseQuery { gte: '{{ column }} >= {{ param }}', lt: '{{ column }} < {{ param }}', lte: '{{ column }} <= {{ param }}', + like_pattern: '{% if start_wild %}\'%\' || {% endif %}{{ value }}{% if end_wild %}|| \'%\'{% endif %}', always_true: '1 == 1' }, diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index a595e0523dc17..4b21b0fde2dab 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -976,7 +976,7 @@ describe('SQL Generation', () => { console.log(query.buildSqlAndParams()); - return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + return dbRunner.testQuery(query.buildSqlAndParamsTest()).then(res => { console.log(JSON.stringify(res)); expect(res).toEqual( [{ visitor_checkins__revenue_per_checkin: '50' }] @@ -1721,7 +1721,7 @@ describe('SQL Generation', () => { ])); it( - 'contains filter', + 'contains filter 1', () => runQueryTest({ measures: [], dimensions: [ diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs index fd1362ddad8ea..36a1543f2035a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs @@ -60,6 +60,7 @@ impl BaseQuery { fn build_sql_and_params_impl(&self) -> Result { if self.request.is_simple_query()? { + println!("!!!! IS SIMPLE"); let planner = SimpleQueryPlanner::new( self.query_tools.clone(), self.request.clone(), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs index 249092a5774f2..fd5cd6e179a77 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs @@ -116,6 +116,12 @@ impl BaseFilter { FilterOperator::Gte => self.gte_where(&member_sql)?, FilterOperator::Lt => self.lt_where(&member_sql)?, FilterOperator::Lte => self.lte_where(&member_sql)?, + FilterOperator::Contains => self.contains_where(&member_sql)?, + FilterOperator::NotContains => self.not_contains_where(&member_sql)?, + FilterOperator::StartsWith => self.starts_with_where(&member_sql)?, + FilterOperator::NotStartsWith => self.not_starts_with_where(&member_sql)?, + FilterOperator::EndsWith => self.ends_with_where(&member_sql)?, + FilterOperator::NotEndsWith => self.not_ends_with_where(&member_sql)?, }; Ok(res) } @@ -243,6 +249,58 @@ impl BaseFilter { .lte(member_sql.to_string(), self.first_param()?) } + fn contains_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, false, true, true) + } + + fn not_contains_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, true, true, true) + } + + fn starts_with_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, false, false, true) + } + + fn not_starts_with_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, true, false, true) + } + + fn ends_with_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, false, true, false) + } + + fn not_ends_with_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, true, true, false) + } + + fn like_or_where( + &self, + member_sql: &str, + not: bool, + start_wild: bool, + end_wild: bool, + ) -> Result { + let values = self.filter_and_allocate_values(); + let like_parts = values + .into_iter() + .map(|v| { + self.templates + .ilike(member_sql, &v, start_wild, end_wild, not) + }) + .collect::, _>>()?; + let logical_symbol = if not { " AND " } else { " OR " }; + let null_check = if self.is_need_null_chek(not) { + self.templates.or_is_null_check(member_sql.to_string())? + } else { + "".to_string() + }; + Ok(format!( + "({}){}", + like_parts.join(logical_symbol), + null_check + )) + } + fn allocate_date_params(&self) -> Result<(String, String), CubeError> { if self.values.len() >= 2 { let from = if let Some(from_str) = &self.values[0] { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs index d66caf4368928..30e0e39012813 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs @@ -15,6 +15,12 @@ pub enum FilterOperator { Gte, Lt, Lte, + Contains, + NotContains, + StartsWith, + NotStartsWith, + NotEndsWith, + EndsWith, } impl FromStr for FilterOperator { @@ -32,6 +38,12 @@ impl FromStr for FilterOperator { "gte" => Ok(Self::Gte), "lt" => Ok(Self::Lt), "lte" => Ok(Self::Lte), + "contains" => Ok(Self::Contains), + "notcontains" => Ok(Self::NotContains), + "startswith" => Ok(Self::StartsWith), + "notstartswith" => Ok(Self::NotStartsWith), + "endswith" => Ok(Self::EndsWith), + "notendswith" => Ok(Self::NotEndsWith), _ => Err(CubeError::user(format!("Unknown filter operator {}", s))), } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs index d6bf872a608e0..7e22d32fa711f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs @@ -33,21 +33,8 @@ impl FullKeyAggregateQueryPlanner { ))); } - let measures = self.query_properties.full_key_aggregate_measures()?; - - let inner_measures = measures - .multiplied_measures - .iter() - .chain(measures.multi_stage_measures.iter()) - .chain(measures.regular_measures.iter()) - .cloned() - .collect_vec(); - - let mut aggregate = self.outer_measures_join_full_key_aggregate( - &inner_measures, - &self.query_properties.measures(), - joins, - )?; + let mut aggregate = + self.outer_measures_join_full_key_aggregate(&self.query_properties.measures(), joins)?; if !ctes.is_empty() { aggregate.set_ctes(ctes.clone()); } @@ -57,7 +44,6 @@ impl FullKeyAggregateQueryPlanner { fn outer_measures_join_full_key_aggregate( &self, - _inner_measures: &Vec>, outer_measures: &Vec>, joins: Vec>, ) -> Result { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index 63465f24c8ef7..a669a991257a4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -381,13 +381,14 @@ impl QueryProperties { } pub fn is_simple_query(&self) -> Result { - for member in self.all_members(false) { - match self.get_symbol_aggregate_type(&member.member_evaluator())? { - SymbolAggregateType::Regular => {} - _ => return Ok(false), - } + let full_aggregate_measure = self.full_key_aggregate_measures()?; + if full_aggregate_measure.multiplied_measures.is_empty() + && full_aggregate_measure.multi_stage_measures.is_empty() + { + Ok(true) + } else { + Ok(false) } - Ok(true) } pub fn should_use_time_series(&self) -> Result { @@ -403,33 +404,23 @@ impl QueryProperties { let mut result = FullKeyAggregateMeasures::default(); let measures = self.measures(); for m in measures.iter() { - match self.get_symbol_aggregate_type(m.member_evaluator())? { - SymbolAggregateType::Regular => result.regular_measures.push(m.clone()), - SymbolAggregateType::Multiplied => result.multiplied_measures.push(m.clone()), - SymbolAggregateType::MultiStage => result.multi_stage_measures.push(m.clone()), + if has_multi_stage_members(m.member_evaluator(), self.ignore_cumulative)? { + result.multi_stage_measures.push(m.clone()) + } else { + for item in + collect_multiplied_measures(self.query_tools.clone(), m.member_evaluator())? + { + if item.multiplied { + println!("!!!! multiplied measure: {:?}", item.measure.full_name()); + result.multiplied_measures.push(item.measure.clone()); + } else { + println!("!!!! regular measure: {:?}", item.measure.full_name()); + result.regular_measures.push(item.measure.clone()); + } + } } } Ok(result) } - - fn get_symbol_aggregate_type( - &self, - symbol: &Rc, - ) -> Result { - let symbol_type = if has_multi_stage_members(symbol, self.ignore_cumulative)? { - SymbolAggregateType::MultiStage - } else if let Some(multiple) = - collect_multiplied_measures(self.query_tools.clone(), symbol)? - { - if multiple.multiplied { - SymbolAggregateType::Multiplied - } else { - SymbolAggregateType::Regular - } - } else { - SymbolAggregateType::Regular - }; - Ok(symbol_type) - } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs index 8ee83c026ec8e..75d4effc5a500 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs @@ -2,31 +2,75 @@ use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::{ EvaluationNode, MemberSymbol, MemberSymbolType, TraversalVisitor, }; +use crate::planner::BaseMeasure; use cubenativeutils::CubeError; +use std::collections::HashSet; use std::rc::Rc; -pub struct RootMeasureResult { +struct CompositeMeasuresCollector { + parent_measure: Option>, + composite_measures: HashSet, +} + +impl CompositeMeasuresCollector { + pub fn new() -> Self { + Self { + parent_measure: None, + composite_measures: HashSet::new(), + } + } + + pub fn extract_result(self) -> HashSet { + self.composite_measures + } +} + +impl TraversalVisitor for CompositeMeasuresCollector { + fn on_node_traverse(&mut self, node: &Rc) -> Result { + let res = match node.symbol() { + MemberSymbolType::Measure(e) => { + if let Some(parent) = &self.parent_measure { + if parent.cube_name() != node.cube_name() { + self.composite_measures.insert(parent.full_name()); + } + } + + self.parent_measure = Some(node.clone()); + true + } + MemberSymbolType::Dimension(_) => false, + _ => false, + }; + Ok(res) + } +} + +pub struct MeasureResult { pub multiplied: bool, - pub measure: String, + pub measure: Rc, } pub struct MultipliedMeasuresCollector { query_tools: Rc, + composite_measures: HashSet, parent_measure: Option, - root_measure: Option, + root_measure: Option, + colllected_measures: Vec, } impl MultipliedMeasuresCollector { - pub fn new(query_tools: Rc) -> Self { + pub fn new(query_tools: Rc, composite_measures: HashSet) -> Self { Self { query_tools, + composite_measures, parent_measure: None, root_measure: None, + colllected_measures: vec![], } } - pub fn extract_result(self) -> Option { - self.root_measure + pub fn extract_result(self) -> Vec { + self.colllected_measures } } @@ -43,16 +87,22 @@ impl TraversalVisitor for MultipliedMeasuresCollector { .unwrap_or(&false) .clone(); - if self.parent_measure.is_none() { - self.root_measure = Some(RootMeasureResult { + if !self.composite_measures.contains(&full_name) { + self.colllected_measures.push(MeasureResult { multiplied, - measure: full_name.clone(), + measure: BaseMeasure::try_new(node.clone(), self.query_tools.clone())? + .unwrap(), }) } - self.parent_measure = Some(full_name); - true + + self.parent_measure = Some(full_name.clone()); + if self.composite_measures.contains(&full_name) { + true + } else { + false + } } - MemberSymbolType::Dimension(_) => true, + MemberSymbolType::Dimension(_) => false, _ => false, }; Ok(res) @@ -62,8 +112,11 @@ impl TraversalVisitor for MultipliedMeasuresCollector { pub fn collect_multiplied_measures( query_tools: Rc, node: &Rc, -) -> Result, CubeError> { - let mut visitor = MultipliedMeasuresCollector::new(query_tools); +) -> Result, CubeError> { + let mut composite_collector = CompositeMeasuresCollector::new(); + composite_collector.apply(node)?; + let composite_measures = composite_collector.extract_result(); + let mut visitor = MultipliedMeasuresCollector::new(query_tools, composite_measures); visitor.apply(node)?; Ok(visitor.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs index 5c53c7833ca61..70ebbf91d2ad1 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs @@ -66,6 +66,10 @@ impl EvaluationNode { self.symbol.name() } + pub fn cube_name(&self) -> String { + self.symbol.cube_name() + } + pub fn is_measure(&self) -> bool { self.symbol.is_measure() } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs index d719fb93492c2..4beabaa483ae1 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs @@ -28,6 +28,16 @@ impl MemberSymbolType { MemberSymbolType::SimpleSql(_) => "".to_string(), } } + + pub fn cube_name(&self) -> String { + match self { + MemberSymbolType::Dimension(d) => d.cube_name().clone(), + MemberSymbolType::Measure(m) => m.cube_name().clone(), + MemberSymbolType::CubeName(c) => c.cube_name().clone(), + MemberSymbolType::CubeTable(c) => c.cube_name().clone(), + MemberSymbolType::SimpleSql(_) => "".to_string(), + } + } pub fn is_measure(&self) -> bool { matches!(self, Self::Measure(_)) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs index c7eaaf9a8ed11..05f9a85af6680 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs @@ -1,7 +1,7 @@ use crate::cube_bridge::sql_templates_render::SqlTemplatesRender; use cubenativeutils::CubeError; use minijinja::context; -use std::rc::Rc; +use std::{ptr::null, rc::Rc}; #[derive(Clone)] pub struct FilterTemplates { @@ -182,11 +182,37 @@ impl FilterTemplates { ) } - fn additional_null_check(&self, need: bool, column: &String) -> Result { + pub fn additional_null_check(&self, need: bool, column: &String) -> Result { if need { self.or_is_null_check(column.clone()) } else { Ok(String::default()) } } + + pub fn ilike( + &self, + column: &str, + value: &str, + start_wild: bool, + end_wild: bool, + not: bool, + ) -> Result { + let pattern = self.render.render_template( + &"filters/like_pattern", + context! { + start_wild => start_wild, + value => value, + end_wild => end_wild + }, + )?; + self.render.render_template( + &"expressions/ilike", + context! { + expr => column.clone(), + negated => not, + pattern => pattern + }, + ) + } }