diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 06fe30f1b5fe3..7fcf5adbd1460 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -3213,25 +3213,35 @@ export class BaseQuery { DATE: 'DATE({{ args_concat }})', }, statements: { - select: 'SELECT {% if distinct %}DISTINCT {% endif %}' + + select: '{% if ctes %} WITH \n' + + '{{ ctes | join(\',\n\') }}\n' + + '{% endif %}' + + 'SELECT {% if distinct %}DISTINCT {% endif %}' + '{{ select_concat | map(attribute=\'aliased\') | join(\', \') }} {% if from %}\n' + 'FROM (\n' + '{{ from | indent(2, true) }}\n' + - ') AS {{ from_alias }}{% endif %}' + + ') AS {{ from_alias }}{% elif from_prepared %}\n' + + 'FROM {{ from_prepared }}' + + '{% endif %}' + '{% if filter %}\nWHERE {{ filter }}{% endif %}' + '{% if group_by %}\nGROUP BY {{ group_by }}{% endif %}' + + '{% if having %}\nHAVING {{ having }}{% endif %}' + '{% if order_by %}\nORDER BY {{ order_by | map(attribute=\'expr\') | join(\', \') }}{% endif %}' + '{% if limit is not none %}\nLIMIT {{ limit }}{% endif %}' + '{% if offset is not none %}\nOFFSET {{ offset }}{% endif %}', group_by_exprs: '{{ group_by | map(attribute=\'index\') | join(\', \') }}', + join: '{{ join_type }} JOIN {{ source }} ON {{ condition }}', + cte: '{{ alias }} AS ({{ query | indent(2, true) }})' }, expressions: { column_reference: '{% if table_name %}{{ table_name }}.{% endif %}{{ name }}', column_aliased: '{{expr}} {{quoted_alias}}', + query_aliased: '{{ query }} AS {{ quoted_alias }}', case: 'CASE{% if expr %} {{ expr }}{% endif %}{% for when, then in when_then %} WHEN {{ when }} THEN {{ then }}{% endfor %}{% if else_expr %} ELSE {{ else_expr }}{% endif %} END', is_null: '{{ expr }} IS {% if negate %}NOT {% endif %}NULL', binary: '({{ left }} {{ op }} {{ right }})', sort: '{{ expr }} {% if asc %}ASC{% else %}DESC{% endif %}{% if nulls_first %} NULLS FIRST{% endif %}', + order_by: '{% if index %} {{ index }} {% else %} {{ expr }} {% endif %} {% if asc %}ASC{% else %}DESC{% endif %}{% if nulls_first %} NULLS FIRST{% endif %}', cast: 'CAST({{ expr }} AS {{ data_type }})', window_function: '{{ fun_call }} OVER ({% if partition_by_concat %}PARTITION BY {{ partition_by_concat }}{% if order_by_concat or window_frame %} {% endif %}{% endif %}{% if order_by_concat %}ORDER BY {{ order_by_concat }}{% if window_frame %} {% endif %}{% endif %}{% if window_frame %}{{ window_frame }}{% endif %})', window_frame_bounds: '{{ frame_type }} BETWEEN {{ frame_start }} AND {{ frame_end }}', @@ -3260,7 +3270,8 @@ export class BaseQuery { gt: '{{ column }} > {{ param }}', gte: '{{ column }} >= {{ param }}', lt: '{{ column }} < {{ param }}', - lte: '{{ column }} <= {{ param }}' + lte: '{{ column }} <= {{ param }}', + always_true: '1 == 1' }, quotes: { @@ -3270,6 +3281,10 @@ export class BaseQuery { params: { param: '?' }, + join_types: { + inner: 'INNER', + left: 'LEFT' + }, window_frame_types: { rows: 'ROWS', range: 'RANGE', diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index cd494f27188a5..8ee333d8fe534 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -538,7 +538,7 @@ describe('SQL Generation', () => { }); `); - it('simple join', async () => { + it('simple join 1', async () => { await compiler.compile(); console.log(joinGraph.buildJoin(['visitor_checkins', 'visitors'])); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/aggregation.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/aggregation.rs deleted file mode 100644 index f2c4506c3e870..0000000000000 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/aggregation.rs +++ /dev/null @@ -1,16 +0,0 @@ -use super::filter::Filter; -use super::select::Select; -use datafusion::logical_expr::Expr; - -pub struct Aggregation { - select: Select, - group_by: Vec, - aggregates: Vec, - having: Option, -} - -pub struct Join { - select: Select, - group_by: Vec, - having: Option, -} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs index 46464b08606d5..f118aefe5d8e5 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs @@ -1,5 +1,6 @@ use super::Schema; use crate::planner::filter::BaseFilter; +use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::VisitorContext; use cubenativeutils::CubeError; use std::fmt; @@ -45,6 +46,7 @@ impl fmt::Display for FilterGroupOperator { impl FilterItem { pub fn to_sql( &self, + templates: &PlanSqlTemplates, context: Rc, schema: Rc, ) -> Result { @@ -54,13 +56,14 @@ impl FilterItem { let items_sql = group .items .iter() - .map(|itm| itm.to_sql(context.clone(), schema.clone())) + .map(|itm| itm.to_sql(templates, context.clone(), schema.clone())) .collect::, _>>()?; - if items_sql.is_empty() { - format!("( 1 = 1 )") + let result = if items_sql.is_empty() { + templates.always_true()? } else { - format!("({})", items_sql.join(&operator)) - } + items_sql.join(&operator) + }; + format!("({})", result) } FilterItem::Item(item) => { let sql = item.to_sql(context.clone(), schema)?; @@ -74,13 +77,14 @@ impl FilterItem { impl Filter { pub fn to_sql( &self, + templates: &PlanSqlTemplates, context: Rc, schema: Rc, ) -> Result { let res = self .items .iter() - .map(|itm| itm.to_sql(context.clone(), schema.clone())) + .map(|itm| itm.to_sql(templates, context.clone(), schema.clone())) .collect::, _>>()? .join(" AND "); Ok(res) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/from.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/from.rs index 2bde1f7b67a13..7da4fb0dbcf39 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/from.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/from.rs @@ -70,10 +70,7 @@ impl SingleAliasedSource { ) -> Result { let sql = self.source.to_sql(templates, context)?; - Ok(format!( - "{sql} AS {}", - templates.quote_identifier(&self.alias)? - )) + templates.query_aliased(&sql, &self.alias) } pub fn make_schema(&self) -> Schema { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs index 8579fa8e2f18a..19e8283bc5e68 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs @@ -68,20 +68,7 @@ impl DimensionJoinCondition { dimension, schema.clone(), )?; - let null_check = if self.null_check { - format!( - " OR ({} AND {})", - templates.is_null_expr(&left_column, false)?, - templates.is_null_expr(&right_column, false)? - ) - } else { - format!("") - }; - - Ok(format!( - "({} = {}{})", - left_column, right_column, null_check - )) + templates.join_by_dimension_conditions(&left_column, &right_column, self.null_check) } fn resolve_member_alias( @@ -157,14 +144,13 @@ impl JoinItem { context: Rc, schema: Rc, ) -> Result { - let operator = if self.is_inner { "INNER" } else { "LEFT" }; let on_sql = self.on.to_sql(templates, context.clone(), schema)?; - Ok(format!( - "{} JOIN {} ON {}", - operator, - self.from.to_sql(templates, context)?, - on_sql - )) + let result = templates.join( + &self.from.to_sql(templates, context)?, + &on_sql, + self.is_inner, + )?; + Ok(result) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/select.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/select.rs index ce88d0c6e78e3..3ea73d2fb136a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/select.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/select.rs @@ -2,6 +2,9 @@ use itertools::Itertools; use super::{Cte, Expr, Filter, From, OrderBy, Schema, SchemaColumn}; use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::sql_templates::{ + TemplateGroupByColumn, TemplateOrderByColumn, TemplateProjectionColumn, +}; use crate::planner::VisitorContext; use cubenativeutils::CubeError; use std::rc::Rc; @@ -21,12 +24,14 @@ impl AliasedExpr { templates: &PlanSqlTemplates, context: Rc, schema: Rc, - ) -> Result { - Ok(format!( - "{} {}", - self.expr.to_sql(templates, context, schema)?, - templates.quote_identifier(&self.alias)? - )) + ) -> Result { + let expr = self.expr.to_sql(templates, context, schema)?; + let aliased = templates.column_aliased(&expr, &self.alias)?; + Ok(TemplateProjectionColumn { + expr, + alias: self.alias.clone(), + aliased, + }) } } @@ -76,90 +81,79 @@ impl Select { .iter() .map(|p| p.to_sql(templates, self.context.clone(), schema.clone())) .collect::, _>>()? - .join(", ") } else { - format!(" * ") + vec![TemplateProjectionColumn { + expr: format!("*"), + alias: format!(""), + aliased: format!("*"), + }] }; let where_condition = if let Some(filter) = &self.filter { - format!( - " WHERE {}", - filter.to_sql(self.context.clone(), schema.clone())? - ) + Some(filter.to_sql(templates, self.context.clone(), schema.clone())?) } else { - format!("") + None }; - let group_by = if !self.group_by.is_empty() { - let str = self - .group_by - .iter() - .enumerate() - .map(|(i, _)| format!("{}", i + 1)) - .join(", "); - format!(" GROUP BY {}", str) - } else { - format!("") - }; + let group_by = self + .group_by + .iter() + .enumerate() + .map(|(i, expr)| -> Result<_, CubeError> { + let expr = expr.to_sql(templates, self.context.clone(), schema.clone())?; + Ok(TemplateGroupByColumn { expr, index: i + 1 }) + }) + .collect::, _>>()?; let having = if let Some(having) = &self.having { - format!( - " HAVING {}", - having.to_sql(self.context.clone(), schema.clone())? - ) + Some(having.to_sql(templates, self.context.clone(), schema.clone())?) } else { - format!("") + None }; - let ctes = if !self.ctes.is_empty() { - let ctes_sql = self - .ctes - .iter() - .map(|cte| -> Result<_, CubeError> { - Ok(format!( - " {} as ({})", - cte.name(), - cte.query().to_sql(templates)? - )) - }) - .collect::, _>>()? - .join(",\n"); - format!("WITH\n{ctes_sql}\n") - } else { - "".to_string() - }; + let ctes = self + .ctes + .iter() + .map(|cte| -> Result<_, CubeError> { + templates.cte(&cte.query().to_sql(templates)?, &cte.name().clone()) + }) + .collect::, _>>()?; - let order_by = if !self.order_by.is_empty() { - let order_sql = self - .order_by - .iter() - .map(|itm| format!("{} {}", itm.pos, itm.asc_str())) - .collect::>() - .join(", "); - format!(" ORDER BY {}", order_sql) - } else { - format!("") - }; + let order_by = self + .order_by + .iter() + .map(|itm| -> Result<_, CubeError> { + let expr = templates.order_by( + &itm.expr + .to_sql(templates, self.context.clone(), schema.clone())?, + Some(itm.pos), + !itm.desc, + )?; + Ok(TemplateOrderByColumn { expr }) + }) + .collect::, _>>()?; - let distinct = if self.is_distinct { "DISTINCT " } else { "" }; let from = self.from.to_sql(templates, self.context.clone())?; - let limit = if let Some(limit) = self.limit { - format!(" LIMIT {limit}") - } else { - format!("") - }; - let offset = if let Some(offset) = self.offset { - format!(" OFFSET {offset}") - } else { - format!("") - }; - let res = format!( + let result = templates.select( + ctes, + &from, + projection, + where_condition, + group_by, + having, + order_by, + self.limit, + self.offset, + self.is_distinct, + )?; + + /* let res = format!( "{ctes}SELECT\ \n {distinct}{projection}\ \n FROM\ \n{from}{where_condition}{group_by}{having}{order_by}{limit}{offset}", - ); - Ok(res) + ); */ + Ok(result) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/mod.rs index 48033f0e4fac3..8e47b7f5a4e2e 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/mod.rs @@ -1,5 +1,7 @@ pub mod filter; pub mod plan; +pub mod structs; pub use filter::FilterTemplates; pub use plan::PlanSqlTemplates; +pub use structs::{TemplateGroupByColumn, TemplateOrderByColumn, TemplateProjectionColumn}; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs index 3da447d5a31c2..aca5695ef8544 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs @@ -1,3 +1,4 @@ +use super::{TemplateGroupByColumn, TemplateOrderByColumn, TemplateProjectionColumn}; use crate::cube_bridge::sql_templates_render::SqlTemplatesRender; use convert_case::{Case, Casing}; use cubenativeutils::CubeError; @@ -42,7 +43,7 @@ impl PlanSqlTemplates { )) } - pub fn alias_expr(&self, expr: &str, alias: &str) -> Result { + pub fn column_aliased(&self, expr: &str, alias: &str) -> Result { let quoted_alias = self.quote_identifier(alias)?; self.render.render_template( "expressions/column_aliased", @@ -73,4 +74,168 @@ impl PlanSqlTemplates { context! { expr => expr, negate => negate }, ) } + pub fn always_true(&self) -> Result { + Ok(self.render.get_template("filters/always_true")?.clone()) + } + + pub fn query_aliased(&self, query: &str, alias: &str) -> Result { + let quoted_alias = self.quote_identifier(alias)?; + self.render.render_template( + "expressions/query_aliased", + context! { query => query, quoted_alias => quoted_alias }, + ) + } + + pub fn order_by( + &self, + expr: &str, + index: Option, + asc: bool, + ) -> Result { + self.render.render_template( + "expressions/order_by", + context! { + expr => expr, + index => index, + asc => asc + }, + ) + } + + pub fn group_by(&self, items: Vec) -> Result { + self.render.render_template( + "statements/group_by_exprs", + context! { + group_by => items + }, + ) + } + + pub fn cte(&self, query: &str, alias: &str) -> Result { + self.render.render_template( + "statements/cte", + context! { + query => query, + alias => alias + }, + ) + } + + pub fn select( + &self, + ctes: Vec, + from: &str, + projection: Vec, + where_condition: Option, + group_by: Vec, + having: Option, + order_by: Vec, + limit: Option, + offset: Option, + distinct: bool, + ) -> Result { + self.render.render_template( + "statements/select", + context! { + from_prepared => from, + select_concat => projection, + group_by => self.group_by(group_by)?, + projection => projection, + order_by => order_by, + filter => where_condition, + having => having, + limit => limit, + offset => offset, + distinct => distinct, + ctes => ctes, + }, + ) + } + + /* pub fn select( + &self, + from: String, + projection: Vec, + group_by: Vec, + group_descs: Vec>, + aggregate: Vec, + alias: String, + filter: Option, + _having: Option, + order_by: Vec, + limit: Option, + offset: Option, + distinct: bool, + ) -> Result { + let group_by = self.to_template_columns(group_by)?; + let aggregate = self.to_template_columns(aggregate)?; + let projection = self.to_template_columns(projection)?; + let order_by = self.to_template_columns(order_by)?; + let select_concat = group_by + .iter() + .chain(aggregate.iter()) + .chain(projection.iter()) + .map(|c| c.clone()) + .collect::>(); + let quoted_from_alias = self.quote_identifier(&alias)?; + let has_grouping_sets = group_descs.iter().any(|d| d.is_some()); + let group_by_expr = if has_grouping_sets { + self.group_by_with_grouping_sets(&group_by, &group_descs)? + } else { + self.render_template( + "statements/group_by_exprs", + context! { group_by => group_by }, + )? + }; + self.render.render_template( + "statements/select", + context! { + from => from, + select_concat => select_concat, + group_by => group_by_expr, + aggregate => aggregate, + projection => projection, + order_by => order_by, + filter => filter, + from_alias => quoted_from_alias, + limit => limit, + offset => offset, + distinct => distinct, + }, + ) + } */ + + pub fn join(&self, source: &str, condition: &str, is_inner: bool) -> Result { + let join_type = if is_inner { + self.render.get_template("join_types/inner")? + } else { + self.render.get_template("join_types/left")? + }; + self.render.render_template( + "statements/join", + context! { source => source, condition => condition, join_type => join_type }, + ) + } + + pub fn join_by_dimension_conditions( + &self, + left_column: &String, + right_column: &String, + null_check: bool, + ) -> Result { + let null_check = if null_check { + format!( + " OR ({} AND {})", + self.is_null_expr(&left_column, false)?, + self.is_null_expr(&right_column, false)? + ) + } else { + format!("") + }; + + Ok(format!( + "({} = {}{})", + left_column, right_column, null_check + )) + } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/structs.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/structs.rs new file mode 100644 index 0000000000000..39924ba8793b4 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/structs.rs @@ -0,0 +1,19 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TemplateProjectionColumn { + pub expr: String, + pub alias: String, + pub aliased: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TemplateGroupByColumn { + pub expr: String, + pub index: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TemplateOrderByColumn { + pub expr: String, +}