From c7432aa1b14db58d501e2698cf6eff0a3b884a1f Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Fri, 29 Sep 2023 15:36:55 -0700 Subject: [PATCH] Add `EXCLUDE` to the parser, ast, plan, and plan schema inferencer (#1226) Co-authored-by: John Ed Quinn <40360967+johnedquinn@users.noreply.github.com> --- CHANGELOG.md | 4 + .../org/partiql/ast/helpers/ToLegacyAst.kt | 46 +- partiql-ast/src/main/pig/partiql.ion | 19 +- .../src/main/resources/partiql_ast.ion | 17 + .../GroupByPathExpressionVisitorTransform.kt | 2 + .../eval/visitors/VisitorTransformBase.kt | 2 + .../lang/planner/PlanningProblemDetails.kt | 6 + .../lang/planner/transforms/plan/PlanTyper.kt | 174 ++- .../lang/planner/transforms/plan/PlanUtils.kt | 1 + .../planner/transforms/plan/RelConverter.kt | 32 + .../planner/transforms/plan/RexConverter.kt | 2 +- .../lang/syntax/impl/PartiQLPigVisitor.kt | 48 + .../PartiQLSchemaInferencerTests.kt | 1368 ++++++++++++++++- .../partiql/lang/syntax/PartiQLParserTest.kt | 369 +++++ partiql-parser/src/main/antlr/PartiQL.g4 | 18 + .../src/main/antlr/PartiQLTokens.g4 | 1 + .../parser/impl/PartiQLParserDefault.kt | 47 +- .../src/main/resources/partiql_plan.ion | 24 + 18 files changed, 2151 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 093aba0b0..29ab462de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ Thank you to all who have contributed! - Support parsing, planning, and evaluation of Bitwise AND operator (&). - The Bitwise And Operator only works for integer operands. - The operator precedence may change based on the pending operator precedence [RFC](https://github.com/partiql/partiql-docs/issues/50). +- **EXPERIMENTAL** Adds `EXCLUDE` to parser, ast, plan, and plan schema inferencer + - This feature is marked as experimental until an RFC is added https://github.com/partiql/partiql-spec/issues/39 + - NOTE: this feature is not currently implemented in the evaluator ### Changed @@ -42,6 +45,7 @@ Thank you to all who have contributed! ### Fixed - Fixes typing of scalar subqueries in the PartiQLSchemaInferencer. Note that usage of `SELECT *` in subqueries is not fully supported. Please make sure to handle InferenceExceptions. +- Fixes schema inferencer behavior for ORDER BY ### Removed diff --git a/partiql-ast/src/main/kotlin/org/partiql/ast/helpers/ToLegacyAst.kt b/partiql-ast/src/main/kotlin/org/partiql/ast/helpers/ToLegacyAst.kt index 66cf52f17..5974d9be2 100644 --- a/partiql-ast/src/main/kotlin/org/partiql/ast/helpers/ToLegacyAst.kt +++ b/partiql-ast/src/main/kotlin/org/partiql/ast/helpers/ToLegacyAst.kt @@ -17,6 +17,7 @@ import com.amazon.ionelement.api.ionSymbol import com.amazon.ionelement.api.metaContainerOf import org.partiql.ast.AstNode import org.partiql.ast.DatetimeField +import org.partiql.ast.Exclude import org.partiql.ast.Expr import org.partiql.ast.From import org.partiql.ast.GraphMatch @@ -665,6 +666,7 @@ private class AstTranslator(val metas: Map) : AstBaseVisi } val project = visitSelect(node.select, ctx) val from = visitFrom(node.from, ctx) + val exclude = node.exclude?.let { visitExclude(it, ctx) } val fromLet = node.let?.let { visitLet(it, ctx) } val where = node.where?.let { visitExpr(it, ctx) } val groupBy = node.groupBy?.let { visitGroupBy(it, ctx) } @@ -672,7 +674,7 @@ private class AstTranslator(val metas: Map) : AstBaseVisi val orderBy = node.orderBy?.let { visitOrderBy(it, ctx) } val limit = node.limit?.let { visitExpr(it, ctx) } val offset = node.offset?.let { visitExpr(it, ctx) } - select(setq, project, from, fromLet, where, groupBy, having, orderBy, limit, offset, metas) + select(setq, project, exclude, from, fromLet, where, groupBy, having, orderBy, limit, offset, metas) } /** @@ -750,6 +752,48 @@ private class AstTranslator(val metas: Map) : AstBaseVisi join(type, lhs, rhs, condition, metas) } + override fun visitExclude(node: Exclude, ctx: Ctx): PartiqlAst.ExcludeOp = translate(node) { metas -> + val excludeExprs = node.exprs.translate(ctx) + excludeOp(excludeExprs, metas) + } + + override fun visitExcludeExcludeExpr(node: Exclude.ExcludeExpr, ctx: Ctx) = translate(node) { metas -> + val root = visitIdentifierSymbol(node.root, ctx) + val steps = node.steps.translate(ctx) + excludeExpr(root = root, steps = steps, metas) + } + + override fun visitExcludeStep(node: Exclude.Step, ctx: Ctx) = + super.visitExcludeStep(node, ctx) as PartiqlAst.ExcludeStep + + override fun visitExcludeStepExcludeTupleAttr(node: Exclude.Step.ExcludeTupleAttr, ctx: Ctx) = translate(node) { metas -> + val attr = node.symbol.symbol + val case = node.symbol.caseSensitivity.toLegacyCaseSensitivity() + excludeTupleAttr(identifier(attr, case), metas) + } + + override fun visitExcludeStepExcludeCollectionIndex( + node: Exclude.Step.ExcludeCollectionIndex, + ctx: Ctx + ) = translate(node) { metas -> + val index = node.index.toLong() + excludeCollectionIndex(index, metas) + } + + override fun visitExcludeStepExcludeTupleWildcard( + node: Exclude.Step.ExcludeTupleWildcard, + ctx: Ctx + ) = translate(node) { metas -> + excludeTupleWildcard(metas) + } + + override fun visitExcludeStepExcludeCollectionWildcard( + node: Exclude.Step.ExcludeCollectionWildcard, + ctx: Ctx + ) = translate(node) { metas -> + excludeCollectionWildcard(metas) + } + override fun visitLet(node: Let, ctx: Ctx) = translate(node) { metas -> val bindings = node.bindings.translate(ctx) let(bindings, metas) diff --git a/partiql-ast/src/main/pig/partiql.ion b/partiql-ast/src/main/pig/partiql.ion index 0eaa7123c..7c845ac44 100644 --- a/partiql-ast/src/main/pig/partiql.ion +++ b/partiql-ast/src/main/pig/partiql.ion @@ -167,6 +167,7 @@ may then be further optimized by selecting better implementations of each operat (select (setq (? set_quantifier)) (project projection) + (exclude_clause (? exclude_op)) (from from_source) (from_let (? let)) (where (? expr)) @@ -197,7 +198,7 @@ may then be further optimized by selecting better implementations of each operat (sum path_step // `someRoot[]`, or `someRoot.someField` which is equivalent to `someRoot['someField']`. (path_expr index::expr case::case_sensitivity) - // `someRoot[*]`] + // `someRoot[*]` (path_wildcard) // `someRoot.*` (path_unpivot)) @@ -220,6 +221,22 @@ may then be further optimized by selecting better implementations of each operat // For ` [AS ]` (project_expr expr::expr as_alias::(? symbol))) + // EXCLUDE clause + (product exclude_op exprs::(* exclude_expr 1)) + + (product exclude_expr root::identifier steps::(* exclude_step 1)) + + (sum exclude_step + // `someRoot.someField` case sensitivity depends on if `someField` is quoted + // `someRoot[] is equivalent to `someRoot.""` (case-sensitive) + (exclude_tuple_attr attr::identifier) + // `someRoot[]` + (exclude_collection_index index::int) + // `someRoot[*]`] + (exclude_tuple_wildcard) + // `someRoot.*` + (exclude_collection_wildcard)) + // A list of LET bindings (product let let_bindings::(* let_binding 1)) diff --git a/partiql-ast/src/main/resources/partiql_ast.ion b/partiql-ast/src/main/resources/partiql_ast.ion index bba3532b0..1c598fcc9 100644 --- a/partiql-ast/src/main/resources/partiql_ast.ion +++ b/partiql-ast/src/main/resources/partiql_ast.ion @@ -494,6 +494,7 @@ expr::[ // The PartiQL `` query expression, think SQL `` s_f_w::{ select: select, // oneof SELECT / SELECT VALUE / PIVOT + exclude: optional::exclude, from: from, let: optional::let, where: optional::expr, @@ -561,6 +562,22 @@ select::[ }, ] +exclude::{ + exprs: list::[exclude_expr], + _: [ + exclude_expr::{ + root: '.identifier.symbol', + steps: list::[step], + }, + step::[ + exclude_tuple_attr::{ symbol: '.identifier.symbol' }, + exclude_collection_index::{ index: int }, + exclude_tuple_wildcard::{}, + exclude_collection_wildcard::{}, + ] + ] +} + // PartiQL FROM Clause Variants — https://partiql.org/dql/from.html from::[ diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/visitors/GroupByPathExpressionVisitorTransform.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/visitors/GroupByPathExpressionVisitorTransform.kt index 42cb5829a..a42ec920f 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/visitors/GroupByPathExpressionVisitorTransform.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/visitors/GroupByPathExpressionVisitorTransform.kt @@ -102,6 +102,7 @@ class GroupByPathExpressionVisitorTransform( val projection = currentAndUnshadowedTransformer.transformExprSelect_project(node) // The scope of the expressions in the FROM clause is the same as that of the parent scope. + val exclude = unshadowedTransformer.transformExprSelect_excludeClause(node) val from = this.transformExprSelect_from(node) val fromLet = unshadowedTransformer.transformExprSelect_fromLet(node) val where = unshadowedTransformer.transformExprSelect_where(node) @@ -116,6 +117,7 @@ class GroupByPathExpressionVisitorTransform( PartiqlAst.Expr.Select( setq = node.setq, project = projection, + excludeClause = exclude, from = from, fromLet = fromLet, where = where, diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/visitors/VisitorTransformBase.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/visitors/VisitorTransformBase.kt index 2c5d9d1de..594a26512 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/visitors/VisitorTransformBase.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/visitors/VisitorTransformBase.kt @@ -36,6 +36,7 @@ abstract class VisitorTransformBase : PartiqlAst.VisitorTransform() { * infinite recursion. */ fun transformExprSelectEvaluationOrder(node: PartiqlAst.Expr.Select): PartiqlAst.Expr { + val exclude = transformExprSelect_excludeClause(node) val from = transformExprSelect_from(node) val fromLet = transformExprSelect_fromLet(node) val where = transformExprSelect_where(node) @@ -51,6 +52,7 @@ abstract class VisitorTransformBase : PartiqlAst.VisitorTransform() { PartiqlAst.Expr.Select( setq = setq, project = project, + excludeClause = exclude, from = from, fromLet = fromLet, where = where, diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/planner/PlanningProblemDetails.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/planner/PlanningProblemDetails.kt index 0fe2e6c0d..27da7d1cb 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/planner/PlanningProblemDetails.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/planner/PlanningProblemDetails.kt @@ -77,6 +77,12 @@ sealed class PlanningProblemDetails( "Please use the `INSERT INTO << , ... >>` form instead." } ) + + data class UnresolvedExcludeExprRoot(val root: String) : + PlanningProblemDetails( + ProblemSeverity.ERROR, + { "Exclude expression given an unresolvable root '$root'" } + ) } private fun quotationHint(caseSensitive: Boolean) = diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/PlanTyper.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/PlanTyper.kt index 68a869381..a909ce76f 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/PlanTyper.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/PlanTyper.kt @@ -37,8 +37,11 @@ import org.partiql.lang.types.TypedOpParameter import org.partiql.lang.types.UnknownArguments import org.partiql.lang.util.cartesianProduct import org.partiql.plan.Arg +import org.partiql.plan.Attribute import org.partiql.plan.Binding import org.partiql.plan.Case +import org.partiql.plan.ExcludeExpr +import org.partiql.plan.ExcludeStep import org.partiql.plan.Plan import org.partiql.plan.PlanNode import org.partiql.plan.Property @@ -155,6 +158,164 @@ internal object PlanTyper : PlanRewriter() { ) } + /** + * Initial implementation of `EXCLUDE` schema inference. Until an RFC is finalized for `EXCLUDE` + * (https://github.com/partiql/partiql-spec/issues/39), this behavior is considered experimental and subject to + * change. + * + * So far this implementation includes + * - Excluding tuple attrs (e.g. t.a.b.c) + * - Excluding tuple wildcards (e.g. t.a.*.b) + * - Excluding collection indexes (e.g. t.a[0].b -- behavior subject to change; see below discussion) + * - Excluding collection wildcards (e.g. t.a[*].b) + * + * There are still discussion points regarding the following edge cases + * - EXCLUDE on a tuple attribute that doesn't exist -- give an error/warning? + * - currently no error + * - EXCLUDE on a tuple attribute that has duplicates -- give an error/warning? exclude one? exclude both? + * - currently excludes both w/ no error + * - EXCLUDE on a collection index as the last step -- mark element type as optional? + * - currently element type as-is + * - EXCLUDE on a collection index w/ remaining path steps -- mark last step's type as optional? + * - currently marks last step's type as optional + * - EXCLUDE on a binding tuple variable (e.g. SELECT ... EXCLUDE t FROM t) -- error? + * - currently a parser error + * - EXCLUDE on a union type -- give an error/warning? no-op? exclude on each type in union? + * - currently exclude on each union type + * - If SELECT list includes an attribute that is excluded, we could consider giving an error in PlanTyper or + * some other semantic pass + * - currently does not give an error + */ + override fun visitRelExclude(node: Rel.Exclude, ctx: Context): Rel.Exclude { + val input = visitRel(node.input, ctx) + val exprs = node.exprs + val typeEnv = input.getTypeEnv() + val newTypeEnv = exprs.fold(typeEnv) { tEnv, expr -> + excludeExpr(tEnv, expr, ctx) + } + return node.copy( + input = input, + common = node.common.copy( + typeEnv = newTypeEnv, + properties = input.getProperties() + ) + ) + } + + private fun attrEqualsExcludeRoot(attr: Attribute, expr: ExcludeExpr): Boolean { + val rootId = expr.root + return attr.name == rootId || (expr.rootCase == Case.INSENSITIVE && attr.name.equals(expr.root, ignoreCase = true)) + } + + private fun excludeExpr(attrs: List, expr: ExcludeExpr, ctx: Context): List { + val resultAttrs = mutableListOf() + val attrsExist = attrs.find { attr -> attrEqualsExcludeRoot(attr, expr) } != null + if (!attrsExist) { + handleUnresolvedExcludeExprRoot(expr.root, ctx) + } + attrs.forEach { attr -> + if (attrEqualsExcludeRoot(attr, expr)) { + if (expr.steps.isEmpty()) { + throw IllegalStateException("Empty `ExcludeExpr.steps` encountered. This should have been caught by the parser.") + } else { + val newType = excludeExprSteps(attr.type, expr.steps, lastStepAsOptional = false, ctx) + resultAttrs.add( + attr.copy( + type = newType + ) + ) + } + } else { + resultAttrs.add( + attr + ) + } + } + return resultAttrs + } + + private fun excludeExprSteps(type: StaticType, steps: List, lastStepAsOptional: Boolean, ctx: Context): StaticType { + fun excludeExprStepsStruct(s: StructType, steps: List, lastStepAsOptional: Boolean): StaticType { + val outputFields = mutableListOf() + val first = steps.first() + s.fields.forEach { field -> + when (first) { + is ExcludeStep.TupleAttr -> { + if (field.key == first.attr || (first.case == Case.INSENSITIVE && field.key.equals(first.attr, ignoreCase = true))) { + if (steps.size == 1) { + if (lastStepAsOptional) { + val newField = StructType.Field(field.key, field.value.asOptional()) + outputFields.add(newField) + } + } else { + outputFields.add(StructType.Field(field.key, excludeExprSteps(field.value, steps.drop(1), lastStepAsOptional, ctx))) + } + } else { + outputFields.add(field) + } + } + is ExcludeStep.TupleWildcard -> { + if (steps.size == 1) { + if (lastStepAsOptional) { + val newField = StructType.Field(field.key, field.value.asOptional()) + outputFields.add(newField) + } + } else { + outputFields.add(StructType.Field(field.key, excludeExprSteps(field.value, steps.drop(1), lastStepAsOptional, ctx))) + } + } + else -> { + // currently no change to field.value and no error thrown; could consider an error/warning in + // the future + outputFields.add(StructType.Field(field.key, field.value)) + } + } + } + return s.copy(fields = outputFields) + } + + fun excludeExprStepsCollection(c: CollectionType, steps: List, lastStepAsOptional: Boolean): StaticType { + var elementType = c.elementType + when (steps.first()) { + is ExcludeStep.CollectionIndex -> { + if (steps.size > 1) { + elementType = excludeExprSteps(elementType, steps.drop(1), lastStepAsOptional = true, ctx) + } + } + is ExcludeStep.CollectionWildcard -> { + if (steps.size > 1) { + elementType = + excludeExprSteps(elementType, steps.drop(1), lastStepAsOptional = lastStepAsOptional, ctx) + } + // currently no change to elementType if collection wildcard is last element; this behavior could + // change based on RFC definition + } + else -> { + // currently no change to elementType and no error thrown; could consider an error/warning in + // the future + } + } + return when (c) { + is BagType -> c.copy(elementType) + is ListType -> c.copy(elementType) + is SexpType -> c.copy(elementType) + } + } + + return when (type) { + is StructType -> excludeExprStepsStruct(type, steps, lastStepAsOptional) + is CollectionType -> excludeExprStepsCollection(type, steps, lastStepAsOptional) + is AnyOfType -> { + StaticType.unionOf( + type.types.map { + excludeExprSteps(it, steps, lastStepAsOptional, ctx) + }.toSet() + ) + } + else -> type + }.flatten() + } + override fun visitRelUnpivot(node: Rel.Unpivot, ctx: Context): Rel.Unpivot { val from = node @@ -227,7 +388,8 @@ internal object PlanTyper : PlanRewriter() { return node.copy( input = input, common = node.common.copy( - typeEnv = typeEnv + typeEnv = typeEnv, + properties = input.getProperties() ) ) } @@ -1026,6 +1188,7 @@ internal object PlanTyper : PlanRewriter() { is Rel.Scan -> this.common is Rel.Sort -> this.common is Rel.Unpivot -> this.common + is Rel.Exclude -> this.common } private fun inferPathComponentExprType( @@ -1707,4 +1870,13 @@ internal object PlanTyper : PlanRewriter() { ) ) } + + private fun handleUnresolvedExcludeExprRoot(root: String, ctx: Context) { + ctx.problemHandler.handleProblem( + Problem( + sourceLocation = UNKNOWN_PROBLEM_LOCATION, + details = PlanningProblemDetails.UnresolvedExcludeExprRoot(root) + ) + ) + } } diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/PlanUtils.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/PlanUtils.kt index 4f33a713f..4da66fcb0 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/PlanUtils.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/PlanUtils.kt @@ -34,6 +34,7 @@ internal object PlanUtils { is Rel.Scan -> input.common.typeEnv is Rel.Sort -> input.common.typeEnv is Rel.Unpivot -> input.common.typeEnv + is Rel.Exclude -> input.common.typeEnv } internal fun Rex.addType(type: StaticType): Rex = when (this) { diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/RelConverter.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/RelConverter.kt index 1ac79a4d4..c0d746929 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/RelConverter.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/RelConverter.kt @@ -4,8 +4,11 @@ import com.amazon.ionelement.api.ionInt import com.amazon.ionelement.api.ionString import org.partiql.lang.domains.PartiqlAst import org.partiql.lang.eval.visitors.VisitorTransformBase +import org.partiql.lang.planner.transforms.plan.RexConverter.convertCase import org.partiql.plan.Binding import org.partiql.plan.Case +import org.partiql.plan.ExcludeExpr +import org.partiql.plan.ExcludeStep import org.partiql.plan.Plan import org.partiql.plan.Rel import org.partiql.plan.Rex @@ -91,6 +94,7 @@ internal class RelConverter { rel = convertHaving(rel, sel.having) rel = convertOrderBy(rel, sel.order) rel = convertFetch(rel, sel.limit, sel.offset) + rel = convertExclude(rel, sel.excludeClause) // append SQL projection if present rel = when (val projection = sel.project) { is PartiqlAst.Projection.ProjectList -> convertProjectList(rel, projection) @@ -100,6 +104,34 @@ internal class RelConverter { return rel } + private fun convertExclude(input: Rel, excludeOp: PartiqlAst.ExcludeOp?): Rel = when (excludeOp) { + null -> input + else -> { + val exprs = excludeOp.exprs.map { convertExcludeExpr(it) } + Plan.relExclude( + common = empty, + input = input, + exprs = exprs, + ) + } + } + + private fun convertExcludeExpr(excludeExpr: PartiqlAst.ExcludeExpr): ExcludeExpr { + val root = excludeExpr.root.name.text + val case = convertCase(excludeExpr.root.case) + val steps = excludeExpr.steps.map { convertExcludeSteps(it) } + return Plan.excludeExpr(root, case, steps) + } + + private fun convertExcludeSteps(excludeStep: PartiqlAst.ExcludeStep): ExcludeStep { + return when (excludeStep) { + is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard -> Plan.excludeStepCollectionWildcard() + is PartiqlAst.ExcludeStep.ExcludeTupleWildcard -> Plan.excludeStepTupleWildcard() + is PartiqlAst.ExcludeStep.ExcludeTupleAttr -> Plan.excludeStepTupleAttr(excludeStep.attr.name.text, convertCase(excludeStep.attr.case)) + is PartiqlAst.ExcludeStep.ExcludeCollectionIndex -> Plan.excludeStepCollectionIndex(excludeStep.index.value.toInt()) + } + } + /** * Appends the appropriate [Rel] operator for the given FROM source */ diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/RexConverter.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/RexConverter.kt index 607996432..633e19c49 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/RexConverter.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/planner/transforms/plan/RexConverter.kt @@ -596,7 +596,7 @@ internal object RexConverter : PartiqlAst.VisitorFold() { } } - private fun convertCase(case: PartiqlAst.CaseSensitivity) = when (case) { + internal fun convertCase(case: PartiqlAst.CaseSensitivity) = when (case) { is PartiqlAst.CaseSensitivity.CaseInsensitive -> Case.INSENSITIVE is PartiqlAst.CaseSensitivity.CaseSensitive -> Case.SENSITIVE } diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/syntax/impl/PartiQLPigVisitor.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/syntax/impl/PartiQLPigVisitor.kt index 2fba1efd7..ee93af85e 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/syntax/impl/PartiQLPigVisitor.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/syntax/impl/PartiQLPigVisitor.kt @@ -36,6 +36,7 @@ import com.amazon.ionelement.api.loadSingleElement import org.antlr.v4.runtime.ParserRuleContext import org.antlr.v4.runtime.Token import org.antlr.v4.runtime.tree.TerminalNode +import org.partiql.ast.Identifier import org.partiql.errors.ErrorCode import org.partiql.errors.Property import org.partiql.errors.PropertyValueMap @@ -513,6 +514,7 @@ internal class PartiQLPigVisitor( override fun visitSfwQuery(ctx: PartiQLParser.SfwQueryContext) = PartiqlAst.build { val projection = visit(ctx.select) as PartiqlAst.Projection val strategy = getSetQuantifierStrategy(ctx.select) + val exclude = ctx.exclude?.let { visitExcludeClause(it) } val from = visitFromClause(ctx.from) val order = ctx.order?.let { visitOrderByClause(it) } val group = ctx.group?.let { visitGroupClause(it) } @@ -524,6 +526,7 @@ internal class PartiQLPigVisitor( val metas = ctx.selectClause().getMetas() select( project = projection, + excludeClause = exclude, from = from, setq = strategy, order = order, @@ -618,6 +621,51 @@ internal class PartiQLPigVisitor( letBinding_(expr, convertSymbolPrimitive(ctx.symbolPrimitive())!!, metas) } + /** + * EXCLUDE CLAUSE + * + */ + override fun visitExcludeClause(ctx: PartiQLParser.ExcludeClauseContext) = PartiqlAst.build { + val excludeExprs = ctx.excludeExpr().map { expr -> + visitExcludeExpr(expr) + } + excludeOp(excludeExprs) + } + + override fun visitExcludeExpr(ctx: PartiQLParser.ExcludeExprContext) = PartiqlAst.build { + val root = visitSymbolPrimitive(ctx.symbolPrimitive()).toIdentifier() + val steps = ctx.excludeExprSteps().map { visit(it) as PartiqlAst.ExcludeStep } + excludeExpr(root, steps) + } + + override fun visitExcludeExprTupleAttr(ctx: PartiQLParser.ExcludeExprTupleAttrContext) = PartiqlAst.build { + val attr = ctx.symbolPrimitive().getString() + val caseSensitivity = when (ctx.symbolPrimitive().ident.type) { + PartiQLParser.IDENTIFIER_QUOTED -> caseSensitive() + PartiQLParser.IDENTIFIER -> caseInsensitive() + else -> throw ParserException("Invalid symbol reference.", ErrorCode.PARSE_INVALID_QUERY) + } + excludeTupleAttr(identifier(attr, caseSensitivity)) + } + + override fun visitExcludeExprCollectionIndex(ctx: PartiQLParser.ExcludeExprCollectionIndexContext) = PartiqlAst.build { + val index = ctx.index.text.toInteger().toLong() + excludeCollectionIndex(index) + } + + override fun visitExcludeExprCollectionAttr(ctx: PartiQLParser.ExcludeExprCollectionAttrContext) = PartiqlAst.build { + val attr = ctx.attr.getStringValue() + excludeTupleAttr(identifier(attr, caseSensitive())) + } + + override fun visitExcludeExprCollectionWildcard(ctx: PartiQLParser.ExcludeExprCollectionWildcardContext) = PartiqlAst.build { + excludeCollectionWildcard() + } + + override fun visitExcludeExprTupleWildcard(ctx: PartiQLParser.ExcludeExprTupleWildcardContext) = PartiqlAst.build { + excludeTupleWildcard() + } + /** * * ORDER BY CLAUSE diff --git a/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt b/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt index 437a60af7..edd60a52f 100644 --- a/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt +++ b/partiql-lang/src/test/kotlin/org/partiql/lang/planner/transforms/PartiQLSchemaInferencerTests.kt @@ -77,19 +77,19 @@ class PartiQLSchemaInferencerTests { val TYPE_BOOL = BOOL private val TYPE_AWS_DDB_PETS_ID = INT private val TYPE_AWS_DDB_PETS_BREED = STRING - val TABLE_AWS_DDB_PETS = BagType( - elementType = StructType( - fields = mapOf( - "id" to TYPE_AWS_DDB_PETS_ID, - "breed" to TYPE_AWS_DDB_PETS_BREED - ), - contentClosed = true, - constraints = setOf( - TupleConstraint.Open(false), - TupleConstraint.UniqueAttrs(true), - TupleConstraint.Ordered - ) - ) + val TABLE_AWS_DDB_PETS_ELEMENT_TYPE = StructType( + fields = mapOf( + "id" to TYPE_AWS_DDB_PETS_ID, + "breed" to TYPE_AWS_DDB_PETS_BREED + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + val TABLE_AWS_DDB_PETS_BAG = BagType( + elementType = TABLE_AWS_DDB_PETS_ELEMENT_TYPE + ) + val TABLE_AWS_DDB_PETS_LIST = ListType( + elementType = TABLE_AWS_DDB_PETS_ELEMENT_TYPE ) val TABLE_AWS_DDB_B = BagType( StructType( @@ -231,14 +231,14 @@ class PartiQLSchemaInferencerTests { catalog = CATALOG_AWS, catalogPath = listOf("ddb"), query = "SELECT * FROM pets", - expected = TABLE_AWS_DDB_PETS + expected = TABLE_AWS_DDB_PETS_BAG ), SuccessTestCase( name = "Project all implicitly", catalog = CATALOG_AWS, catalogPath = listOf("ddb"), query = "SELECT id, breed FROM pets", - expected = TABLE_AWS_DDB_PETS + expected = TABLE_AWS_DDB_PETS_BAG ), SuccessTestCase( name = "Test #4", @@ -286,7 +286,7 @@ class PartiQLSchemaInferencerTests { name = "Test #8", catalog = CATALOG_AWS, query = "SELECT * FROM ddb.pets", - expected = TABLE_AWS_DDB_PETS + expected = TABLE_AWS_DDB_PETS_BAG ), SuccessTestCase( name = "Test #9", @@ -324,7 +324,7 @@ class PartiQLSchemaInferencerTests { SuccessTestCase( name = "Test #14", query = "SELECT * FROM aws.ddb.pets", - expected = TABLE_AWS_DDB_PETS + expected = TABLE_AWS_DDB_PETS_BAG ), SuccessTestCase( name = "Test #15", @@ -636,35 +636,35 @@ class PartiQLSchemaInferencerTests { catalog = CATALOG_AWS, catalogPath = listOf("ddb"), query = "SELECT * FROM pets ORDER BY id", - expected = TABLE_AWS_DDB_PETS + expected = TABLE_AWS_DDB_PETS_LIST ), SuccessTestCase( name = "ORDER BY str", catalog = CATALOG_AWS, catalogPath = listOf("ddb"), query = "SELECT * FROM pets ORDER BY breed", - expected = TABLE_AWS_DDB_PETS + expected = TABLE_AWS_DDB_PETS_LIST ), SuccessTestCase( name = "ORDER BY str", catalog = CATALOG_AWS, catalogPath = listOf("ddb"), query = "SELECT * FROM pets ORDER BY unknown_col", - expected = TABLE_AWS_DDB_PETS + expected = TABLE_AWS_DDB_PETS_LIST ), SuccessTestCase( name = "LIMIT INT", catalog = CATALOG_AWS, catalogPath = listOf("ddb"), query = "SELECT * FROM pets LIMIT 5", - expected = TABLE_AWS_DDB_PETS + expected = TABLE_AWS_DDB_PETS_BAG ), ErrorTestCase( name = "LIMIT STR", catalog = CATALOG_AWS, catalogPath = listOf("ddb"), query = "SELECT * FROM pets LIMIT '5'", - expected = TABLE_AWS_DDB_PETS, + expected = TABLE_AWS_DDB_PETS_BAG, problemHandler = assertProblemExists { Problem( UNKNOWN_PROBLEM_LOCATION, @@ -677,14 +677,14 @@ class PartiQLSchemaInferencerTests { catalog = CATALOG_AWS, catalogPath = listOf("ddb"), query = "SELECT * FROM pets LIMIT 1 OFFSET 5", - expected = TABLE_AWS_DDB_PETS + expected = TABLE_AWS_DDB_PETS_BAG ), ErrorTestCase( name = "OFFSET STR", catalog = CATALOG_AWS, catalogPath = listOf("ddb"), query = "SELECT * FROM pets LIMIT 1 OFFSET '5'", - expected = TABLE_AWS_DDB_PETS, + expected = TABLE_AWS_DDB_PETS_BAG, problemHandler = assertProblemExists { Problem( UNKNOWN_PROBLEM_LOCATION, @@ -1344,6 +1344,1326 @@ class PartiQLSchemaInferencerTests { ) } ), + // EXCLUDE test cases + SuccessTestCase( + name = "EXCLUDE SELECT star", + query = """SELECT * EXCLUDE c.ssn FROM [ + { + 'name': 'Alan', + 'custId': 1, + 'address': { + 'city': 'Seattle', + 'zipcode': 98109, + 'street': '123 Seaplane Dr.' + }, + 'ssn': 123456789 + } + ] AS c""", + expected = BagType( + StructType( + fields = mapOf( + "name" to StaticType.STRING, + "custId" to StaticType.INT, + "address" to StructType( + fields = mapOf( + "city" to StaticType.STRING, + "zipcode" to StaticType.INT, + "street" to StaticType.STRING, + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE SELECT star multiple paths", + query = """SELECT * EXCLUDE c.ssn, c.address.street FROM [ + { + 'name': 'Alan', + 'custId': 1, + 'address': { + 'city': 'Seattle', + 'zipcode': 98109, + 'street': '123 Seaplane Dr.' + }, + 'ssn': 123456789 + } + ] AS c""", + expected = BagType( + StructType( + fields = mapOf( + "name" to StaticType.STRING, + "custId" to StaticType.INT, + "address" to StructType( + fields = mapOf( + "city" to StaticType.STRING, + "zipcode" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE SELECT star list index and list index field", + query = """SELECT * + EXCLUDE + t.a.b.c[0], + t.a.b.c[1].field + FROM [{ + 'a': { + 'b': { + 'c': [ + { + 'field': 0 -- c[0] + }, + { + 'field': 1 -- c[1] + }, + { + 'field': 2 -- c[2] + } + ] + } + }, + 'foo': 'bar' + }] AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "c" to ListType( + elementType = StructType( + fields = mapOf( + "field" to AnyOfType( + setOf( + StaticType.INT, + StaticType.MISSING // c[1]'s `field` was excluded + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + "foo" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE SELECT star collection index as last step", + query = """SELECT * + EXCLUDE + t.a.b.c[0] + FROM [{ + 'a': { + 'b': { + 'c': [0, 1, 2] + } + }, + 'foo': 'bar' + }] AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "c" to ListType( + elementType = StaticType.INT + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + "foo" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + // EXCLUDE regression test (behavior subject to change pending RFC) + SuccessTestCase( + name = "EXCLUDE SELECT star collection wildcard as last step", + query = """SELECT * + EXCLUDE + t.a[*] + FROM [{ + 'a': [0, 1, 2] + }] AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to ListType( + elementType = StaticType.INT // empty list but still preserve typing information + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE SELECT star list wildcard", + query = """SELECT * + EXCLUDE + t.a.b.c[*].field_x + FROM [{ + 'a': { + 'b': { + 'c': [ + { -- c[0] + 'field_x': 0, + 'field_y': 0 + }, + { -- c[1] + 'field_x': 1, + 'field_y': 1 + }, + { -- c[2] + 'field_x': 2, + 'field_y': 2 + } + ] + } + }, + 'foo': 'bar' + }] AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "c" to ListType( + elementType = StructType( + fields = mapOf( + "field_y" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + "foo" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE SELECT star list tuple wildcard", + query = """SELECT * + EXCLUDE + t.a.b.c[*].* + FROM [{ + 'a': { + 'b': { + 'c': [ + { -- c[0] + 'field_x': 0, + 'field_y': 0 + }, + { -- c[1] + 'field_x': 1, + 'field_y': 1 + }, + { -- c[2] + 'field_x': 2, + 'field_y': 2 + } + ] + } + }, + 'foo': 'bar' + }] AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "c" to ListType( + elementType = StructType( + fields = mapOf( + // all fields gone + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + "foo" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE SELECT star order by", + query = """SELECT * + EXCLUDE + t.a + FROM [ + { + 'a': 2, + 'foo': 'bar2' + }, + { + 'a': 1, + 'foo': 'bar1' + }, + { + 'a': 3, + 'foo': 'bar3' + } + ] AS t + ORDER BY t.a""", + expected = ListType( + StructType( + fields = mapOf( + "foo" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE SELECT star with JOINs", + query = """SELECT * + EXCLUDE bar.d + FROM + << + {'a': 1, 'b': 11}, + {'a': 2, 'b': 22} + >> AS foo, + << + {'c': 3, 'd': 33}, + {'c': 4, 'd': 44} + >> AS bar""", + expected = BagType( + StructType( + fields = mapOf( + "a" to StaticType.INT, + "b" to StaticType.INT, + "c" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "SELECT t.b EXCLUDE ex 1", + query = """SELECT t.b EXCLUDE t.b[*].b_1 + FROM << + { + 'a': {'a_1':1,'a_2':2}, + 'b': [ {'b_1':3,'b_2':4}, {'b_1':5,'b_2':6} ], + 'c': 7, + 'd': 8 + } >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "b" to ListType( + elementType = StructType( + fields = mapOf( + "b_2" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "SELECT * EXCLUDE ex 2", + query = """SELECT * EXCLUDE t.b[*].b_1 + FROM << + { + 'a': {'a_1':1,'a_2':2}, + 'b': [ {'b_1':3,'b_2':4}, {'b_1':5,'b_2':6} ], + 'c': 7, + 'd': 8 + } >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "a_1" to StaticType.INT, + "a_2" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + "b" to ListType( + elementType = StructType( + fields = mapOf( + "b_2" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + "c" to StaticType.INT, + "d" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "SELECT VALUE t.b EXCLUDE", + query = """SELECT VALUE t.b EXCLUDE t.b[*].b_1 + FROM << + { + 'a': {'a_1':1,'a_2':2}, + 'b': [ {'b_1':3,'b_2':4}, {'b_1':5,'b_2':6} ], + 'c': 7, + 'd': 8 + } >> AS t""", + expected = BagType( + ListType( + elementType = StructType( + fields = mapOf( + "b_2" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + ) + ), + SuccessTestCase( + name = "SELECT * EXCLUDE collection wildcard and nested tuple attr", + query = """SELECT * EXCLUDE t.a[*].b.c + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': 'zero' } }, + { 'b': { 'c': 1, 'd': 'one' } }, + { 'b': { 'c': 2, 'd': 'two' } } + ] + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to ListType( + elementType = StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "d" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "SELECT * EXCLUDE collection index and nested tuple attr", + query = """SELECT * EXCLUDE t.a[1].b.c + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': 'zero' } }, + { 'b': { 'c': 1, 'd': 'one' } }, + { 'b': { 'c': 2, 'd': 'two' } } + ] + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to ListType( + elementType = StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "c" to StaticType.INT.asOptional(), + "d" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "SELECT * EXCLUDE collection wildcard and nested tuple wildcard", + query = """SELECT * EXCLUDE t.a[*].b.* + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': 'zero' } }, + { 'b': { 'c': 1, 'd': 'one' } }, + { 'b': { 'c': 2, 'd': 'two' } } + ] + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to ListType( + elementType = StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf(), // empty map; all fields of b excluded + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "SELECT * EXCLUDE collection index and nested tuple wildcard", + query = """SELECT * EXCLUDE t.a[1].b.* + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': 'zero' } }, + { 'b': { 'c': 1, 'd': 'one' } }, + { 'b': { 'c': 2, 'd': 'two' } } + ] + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to ListType( + elementType = StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( // all fields of b optional + "c" to StaticType.INT.asOptional(), + "d" to StaticType.STRING.asOptional() + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "SELECT * EXCLUDE collection wildcard and nested collection wildcard", + query = """SELECT * EXCLUDE t.a[*].b.d[*].e + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': [{'e': 'zero', 'f': true}] } }, + { 'b': { 'c': 1, 'd': [{'e': 'one', 'f': true}] } }, + { 'b': { 'c': 2, 'd': [{'e': 'two', 'f': true}] } } + ] + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to ListType( + elementType = StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "c" to StaticType.INT, + "d" to ListType( + elementType = StructType( + fields = mapOf( + "f" to StaticType.BOOL + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "SELECT * EXCLUDE collection index and nested collection wildcard", + query = """SELECT * EXCLUDE t.a[1].b.d[*].e + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': [{'e': 'zero', 'f': true}] } }, + { 'b': { 'c': 1, 'd': [{'e': 'one', 'f': true}] } }, + { 'b': { 'c': 2, 'd': [{'e': 'two', 'f': true}] } } + ] + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to ListType( + elementType = StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "c" to StaticType.INT, + "d" to ListType( + elementType = StructType( + fields = mapOf( + "e" to StaticType.STRING.asOptional(), // last step is optional since only a[1]... is excluded + "f" to StaticType.BOOL + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "SELECT * EXCLUDE collection index and nested collection index", + query = """SELECT * EXCLUDE t.a[1].b.d[0].e + FROM << + { + 'a': [ + { 'b': { 'c': 0, 'd': [{'e': 'zero', 'f': true}] } }, + { 'b': { 'c': 1, 'd': [{'e': 'one', 'f': true}] } }, + { 'b': { 'c': 2, 'd': [{'e': 'two', 'f': true}] } } + ] + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to ListType( + elementType = StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "c" to StaticType.INT, + "d" to ListType( + elementType = StructType( + fields = mapOf( // same as above + "e" to StaticType.STRING.asOptional(), + "f" to StaticType.BOOL + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE case sensitive lookup", + query = """SELECT * EXCLUDE t."a".b['c'] + FROM << + { + 'a': { + 'B': { + 'c': 0, + 'd': 'foo' + } + } + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "B" to StructType( + fields = mapOf( + "d" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE case sensitive lookup with capitalized and uncapitalized attr", + query = """SELECT * EXCLUDE t."a".b['c'] + FROM << + { + 'a': { + 'B': { + 'c': 0, + 'C': true, + 'd': 'foo' + } + } + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "B" to StructType( + fields = mapOf( + "C" to StaticType.BOOL, // keep 'C' + "d" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE case sensitive lookup with both capitalized and uncapitalized removed", + query = """SELECT * EXCLUDE t."a".b.c + FROM << + { + 'a': { + 'B': { -- both 'c' and 'C' to be removed + 'c': 0, + 'C': true, + 'd': 'foo' + } + } + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "B" to StructType( + fields = mapOf( + "d" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE with both duplicates", + query = """SELECT * EXCLUDE t."a".b.c + FROM << + { + 'a': { + 'B': { + 'c': 0, + 'c': true, + 'd': 'foo' + } + } + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "B" to StructType( + fields = mapOf( + // both "c" removed + "d" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(false)) // UniqueAttrs set to false + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + // EXCLUDE regression test (behavior subject to change pending RFC) + SuccessTestCase( + name = "EXCLUDE with removed attribute later referenced", + query = "SELECT * EXCLUDE t.a, t.a.b FROM << { 'a': { 'b': 1 }, 'c': 2 } >> AS t", + expected = BagType( + StructType( + fields = mapOf( + "c" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + // EXCLUDE regression test (behavior subject to change pending RFC) + SuccessTestCase( + name = "EXCLUDE with non-existent attribute reference", + query = "SELECT * EXCLUDE t.attr_does_not_exist FROM << { 'a': 1 } >> AS t", + expected = BagType( + StructType( + fields = mapOf( + "a" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + // EXCLUDE regression test (behavior subject to change pending RFC); could give error/warning + SuccessTestCase( + name = "exclude union of types", + query = """SELECT t EXCLUDE t.a.b + FROM << + { + 'a': { + 'b': 1, -- `b` to be excluded + 'c': 'foo' + } + }, + { + 'a': NULL + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "t" to StaticType.unionOf( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "c" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + StructType( + fields = mapOf( + "a" to StaticType.NULL + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "exclude union of types exclude same type", + query = """SELECT t EXCLUDE t.a.b + FROM << + { + 'a': { + 'b': 1, -- `b` to be excluded + 'c': 'foo' + } + }, + { + 'a': { + 'b': 1, -- `b` to be excluded + 'c': NULL + } + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "t" to StaticType.unionOf( + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "c" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "c" to StaticType.NULL + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "exclude union of types exclude different type", + query = """SELECT t EXCLUDE t.a.c + FROM << + { + 'a': { + 'b': 1, + 'c': 'foo' -- `c` to be excluded + } + }, + { + 'a': { + 'b': 1, + 'c': NULL -- `c` to be excluded + } + } + >> AS t""", + expected = BagType( + StructType( + fields = mapOf( + "t" to StructType( // union gone + fields = mapOf( + "a" to StructType( + fields = mapOf( + "b" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + // EXCLUDE regression test (behavior subject to change pending RFC); could give error/warning + SuccessTestCase( + name = "invalid exclude collection wildcard", + query = """SELECT * EXCLUDE t.a[*] + FROM << + { + 'a': { + 'b': { + 'c': 0, + 'd': 'foo' + } + } + } + >> AS t""", + expected = BagType( + elementType = StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "c" to StaticType.INT, + "d" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + // EXCLUDE regression test (behavior subject to change pending RFC); could give error/warning + SuccessTestCase( + name = "invalid exclude collection index", + query = """SELECT * EXCLUDE t.a[1] + FROM << + { + 'a': { + 'b': { + 'c': 0, + 'd': 'foo' + } + } + } + >> AS t""", + expected = BagType( + elementType = StructType( + fields = mapOf( + "a" to StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "c" to StaticType.INT, + "d" to StaticType.STRING + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + // EXCLUDE regression test (behavior subject to change pending RFC); could give error/warning + SuccessTestCase( + name = "invalid exclude tuple attr", + query = """SELECT * EXCLUDE t.a.b + FROM << + { + 'a': [ + { 'b': 0 }, + { 'b': 1 }, + { 'b': 2 } + ] + } + >> AS t""", + expected = BagType( + elementType = StructType( + fields = mapOf( + "a" to ListType( + elementType = StructType( + fields = mapOf( + "b" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + // EXCLUDE regression test (behavior subject to change pending RFC); could give error/warning + SuccessTestCase( + name = "invalid exclude tuple wildcard", + query = """SELECT * EXCLUDE t.a.* + FROM << + { + 'a': [ + { 'b': 0 }, + { 'b': 1 }, + { 'b': 2 } + ] + } + >> AS t""", + expected = BagType( + elementType = StructType( + fields = mapOf( + "a" to ListType( + elementType = StructType( + fields = mapOf( + "b" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + // EXCLUDE regression test (behavior subject to change pending RFC); could give error/warning + SuccessTestCase( + name = "invalid exclude tuple attr step", + query = """SELECT * EXCLUDE t.b -- `t.b` does not exist + FROM << + { + 'a': << + { 'b': 0 }, + { 'b': 1 }, + { 'b': 2 } + >> + } + >> AS t""", + expected = BagType( + elementType = StructType( + fields = mapOf( + "a" to BagType( + elementType = StructType( + fields = mapOf( + "b" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + // EXCLUDE regression test (behavior subject to change pending RFC); could give error/warning + ErrorTestCase( + name = "invalid exclude root", + query = """SELECT * EXCLUDE nonsense.b -- `nonsense` does not exist in binding tuples + FROM << + { + 'a': << + { 'b': 0 }, + { 'b': 1 }, + { 'b': 2 } + >> + } + >> AS t""", + expected = BagType( + elementType = StructType( + fields = mapOf( + "a" to BagType( + elementType = StructType( + fields = mapOf( + "b" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ), + problemHandler = assertProblemExists { + Problem( + UNKNOWN_PROBLEM_LOCATION, + PlanningProblemDetails.UnresolvedExcludeExprRoot("nonsense") + ) + } + ), + // EXCLUDE regression test (behavior subject to change pending RFC); could give error/warning + SuccessTestCase( + name = "exclude with unions and last step collection index", + query = """SELECT * EXCLUDE t.a[0].c -- `c`'s type to be unioned with `MISSING` + FROM << + { + 'a': [ + { + 'b': 0, + 'c': 0 + }, + { + 'b': 1, + 'c': NULL + }, + { + 'b': 2, + 'c': 0.1 + } + ] + } + >> AS t""", + expected = BagType( + elementType = StructType( + fields = mapOf( + "a" to ListType( + elementType = StaticType.unionOf( + StructType( + fields = mapOf( + "b" to StaticType.INT, + "c" to StaticType.INT.asOptional() + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + StructType( + fields = mapOf( + "b" to StaticType.INT, + "c" to StaticType.NULL.asOptional() + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ), + StructType( + fields = mapOf( + "b" to StaticType.INT, + "c" to StaticType.DECIMAL.asOptional() + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true)) + ) + ) + ) + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), + SuccessTestCase( + name = "EXCLUDE using a catalog", + catalog = CATALOG_B, + query = "SELECT * EXCLUDE t.c FROM b.b.b AS t", + expected = BagType( + elementType = StructType( + fields = mapOf( + "b" to StructType( + fields = mapOf( + "b" to StaticType.INT + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ), + ), + contentClosed = true, + constraints = setOf(TupleConstraint.Open(false), TupleConstraint.UniqueAttrs(true), TupleConstraint.Ordered) + ) + ) + ), SuccessTestCase( name = "BITWISE_AND_1", query = "1 & 2", diff --git a/partiql-lang/src/test/kotlin/org/partiql/lang/syntax/PartiQLParserTest.kt b/partiql-lang/src/test/kotlin/org/partiql/lang/syntax/PartiQLParserTest.kt index 04f7a5088..ebc368b2c 100644 --- a/partiql-lang/src/test/kotlin/org/partiql/lang/syntax/PartiQLParserTest.kt +++ b/partiql-lang/src/test/kotlin/org/partiql/lang/syntax/PartiQLParserTest.kt @@ -4521,6 +4521,375 @@ class PartiQLParserTest : PartiQLParserTestBase() { ) } + // EXCLUDE tests + @Test + fun selectStarExcludeAttrs() = assertExpression( + "SELECT * EXCLUDE t.a, t.b, t.c FROM t" + ) { + select( + project = projectStar(), + excludeClause = excludeOp( + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("a", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("b", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("c", caseInsensitive())) + ), + ) + ), + from = scan(id("t")) + ) + } + + @Test + fun selectListExcludeAttrs() = assertExpression( + "SELECT x, y, z EXCLUDE t.a, t.b, t.c FROM t" + ) { + select( + project = projectList( + projectItems = listOf( + projectExpr( + id("x"), + ), + projectExpr( + id("y"), + ), + projectExpr( + id("z"), + ), + ), + ), + excludeClause = excludeOp( + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("a", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("b", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("c", caseInsensitive())) + ), + ) + ), + from = scan(id("t")) + ) + } + + @Test + fun selectValueExcludeAttrs() = assertExpression( + "SELECT VALUE { 'x': 1, 'y': 2, 'z': 3 } EXCLUDE t.a, t.b, t.c FROM t" + ) { + select( + project = projectValue( + value = struct( + exprPair(lit(ionString("x")), lit(ionInt(1))), + exprPair(lit(ionString("y")), lit(ionInt(2))), + exprPair(lit(ionString("z")), lit(ionInt(3))) + ) + ), + excludeClause = excludeOp( + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("a", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("b", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("c", caseInsensitive())) + ), + ) + ), + from = scan(id("t")) + ) + } + + @Test + fun selectStarExcludeNestedAttrs() = assertExpression( + "SELECT * EXCLUDE t.a.foo.bar, t.b[0].*[*].baz, t.c.d.*.e[*].f.* FROM t" + ) { + select( + project = projectStar(), + excludeClause = excludeOp( + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("a", caseInsensitive())), + excludeTupleAttr(identifier("foo", caseInsensitive())), + excludeTupleAttr(identifier("bar", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("b", caseInsensitive())), + excludeCollectionIndex(0), + excludeTupleWildcard(), + excludeCollectionWildcard(), + excludeTupleAttr(identifier("baz", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("c", caseInsensitive())), + excludeTupleAttr(identifier("d", caseInsensitive())), + excludeTupleWildcard(), + excludeTupleAttr(identifier("e", caseInsensitive())), + excludeCollectionWildcard(), + excludeTupleAttr(identifier("f", caseInsensitive())), + excludeTupleWildcard() + ), + ) + ), + from = scan(id("t")) + ) + } + + @Test + fun selectListExcludeNestedAttrs() = assertExpression( + "SELECT x, y, z EXCLUDE t.a.foo.bar, t.b[0].*[*].baz, t.c.d.*.e[*].f.* FROM t" + ) { + select( + project = projectList( + projectItems = listOf( + projectExpr( + id("x"), + ), + projectExpr( + id("y"), + ), + projectExpr( + id("z"), + ), + ), + ), + excludeClause = excludeOp( + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("a", caseInsensitive())), + excludeTupleAttr(identifier("foo", caseInsensitive())), + excludeTupleAttr(identifier("bar", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("b", caseInsensitive())), + excludeCollectionIndex(0), + excludeTupleWildcard(), + excludeCollectionWildcard(), + excludeTupleAttr(identifier("baz", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("c", caseInsensitive())), + excludeTupleAttr(identifier("d", caseInsensitive())), + excludeTupleWildcard(), + excludeTupleAttr(identifier("e", caseInsensitive())), + excludeCollectionWildcard(), + excludeTupleAttr(identifier("f", caseInsensitive())), + excludeTupleWildcard() + ), + ) + ), + from = scan(id("t")) + ) + } + + @Test + fun selectValueExcludeNestedAttrs() = assertExpression( + "SELECT VALUE { 'x': 1, 'y': 2, 'z': 3 } EXCLUDE t.a.foo.bar, t.b[0].*[*].baz, t.c.d.*.e[*].f.* FROM t" + ) { + select( + project = projectValue( + value = struct( + exprPair(lit(ionString("x")), lit(ionInt(1))), + exprPair(lit(ionString("y")), lit(ionInt(2))), + exprPair(lit(ionString("z")), lit(ionInt(3))) + ) + ), + excludeClause = excludeOp( + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("a", caseInsensitive())), + excludeTupleAttr(identifier("foo", caseInsensitive())), + excludeTupleAttr(identifier("bar", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("b", caseInsensitive())), + excludeCollectionIndex(0), + excludeTupleWildcard(), + excludeCollectionWildcard(), + excludeTupleAttr(identifier("baz", caseInsensitive())) + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("c", caseInsensitive())), + excludeTupleAttr(identifier("d", caseInsensitive())), + excludeTupleWildcard(), + excludeTupleAttr(identifier("e", caseInsensitive())), + excludeCollectionWildcard(), + excludeTupleAttr(identifier("f", caseInsensitive())), + excludeTupleWildcard() + ), + ) + ), + from = scan(id("t")) + ) + } + + @Test + fun selectStarExcludeCaseSensitiveAndInsensitiveAttrs() = assertExpression( + """SELECT * EXCLUDE t.a."b".C['d']."E" FROM t""" + ) { + select( + project = projectStar(), + excludeClause = excludeOp( + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("a", caseInsensitive())), + excludeTupleAttr(identifier("b", caseSensitive())), + excludeTupleAttr(identifier("C", caseInsensitive())), + excludeTupleAttr(identifier("d", caseSensitive())), + excludeTupleAttr(identifier("E", caseSensitive())), + ), + ), + ), + from = scan(id("t")) + ) + } + + @Test + fun pivotExclude() = assertExpression( + """PIVOT v AT attr EXCLUDE t.a[*].b.c.*.d, t.foo.bar[*] FROM t""" + ) { + select( + project = projectPivot( + key = id("v"), + value = id("attr"), + ), + excludeClause = excludeOp( + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("a", caseInsensitive())), + excludeCollectionWildcard(), + excludeTupleAttr(identifier("b", caseInsensitive())), + excludeTupleAttr(identifier("c", caseInsensitive())), + excludeTupleWildcard(), + excludeTupleAttr(identifier("d", caseInsensitive())), + ), + ), + excludeExpr( + root = identifier("t", caseInsensitive()), + steps = listOf( + excludeTupleAttr(identifier("foo", caseInsensitive())), + excludeTupleAttr(identifier("bar", caseInsensitive())), + excludeCollectionWildcard(), + ), + ), + ), + from = scan(id("t")) + ) + } + + @Test + fun selectStarExcludeErrorBinding() = checkInputThrowingParserException( + "SELECT * EXCLUDE t FROM t", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + expectErrorContextValues = mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 20L, + Property.TOKEN_DESCRIPTION to PartiQLParser.FROM.getAntlrDisplayString(), + Property.TOKEN_VALUE to ION.newSymbol("FROM") + ) + ) + + @Test + fun selectStarExcludeErrorBindingWithJoin() = checkInputThrowingParserException( + "SELECT * EXCLUDE t FROM s, t", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + expectErrorContextValues = mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 20L, + Property.TOKEN_DESCRIPTION to PartiQLParser.FROM.getAntlrDisplayString(), + Property.TOKEN_VALUE to ION.newSymbol("FROM") + ) + ) + + @Test + fun selectStarExcludeErrorTrailingComma() = checkInputThrowingParserException( + "SELECT * EXCLUDE t.a.b.c, t.d.e, FROM t", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + expectErrorContextValues = mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 34L, + Property.TOKEN_DESCRIPTION to PartiQLParser.FROM.getAntlrDisplayString(), + Property.TOKEN_VALUE to ION.newSymbol("FROM") + ) + ) + + @Test + fun selectStarExcludeErrorStar() = checkInputThrowingParserException( + "SELECT * EXCLUDE * FROM", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + expectErrorContextValues = mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 18L, + Property.TOKEN_DESCRIPTION to PartiQLParser.ASTERISK.getAntlrDisplayString(), + Property.TOKEN_VALUE to ION.newSymbol("*") + ) + ) + + @Test + fun selectStarExcludeErrorNonLiteralExpr() = checkInputThrowingParserException( + "SELECT * EXCLUDE t.a[x + y] FROM t", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + expectErrorContextValues = mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 22L, + Property.TOKEN_DESCRIPTION to PartiQLParser.IDENTIFIER.getAntlrDisplayString(), + Property.TOKEN_VALUE to ION.newSymbol("x") + ) + ) + @Test fun manyNestedNotPerformanceRegressionTest(): Unit = forEachTarget { val startTime = System.currentTimeMillis() diff --git a/partiql-parser/src/main/antlr/PartiQL.g4 b/partiql-parser/src/main/antlr/PartiQL.g4 index 0e266cb2f..3567e022e 100644 --- a/partiql-parser/src/main/antlr/PartiQL.g4 +++ b/partiql-parser/src/main/antlr/PartiQL.g4 @@ -334,6 +334,23 @@ windowSortSpecList havingClause : HAVING arg=exprSelect; +excludeClause + : EXCLUDE excludeExpr (COMMA excludeExpr)*; + +// Require 1 more `excludeExprSteps` (disallow `EXCLUDE a`). +// There's not a clear use case in which a user would exclude a previously introdced binding variable. If a use case +// arises, we can always change the requirement to 0 or more steps. +excludeExpr + : symbolPrimitive excludeExprSteps+; + +excludeExprSteps + : PERIOD symbolPrimitive # ExcludeExprTupleAttr + | BRACKET_LEFT attr=LITERAL_STRING BRACKET_RIGHT # ExcludeExprCollectionAttr + | BRACKET_LEFT index=LITERAL_INTEGER BRACKET_RIGHT # ExcludeExprCollectionIndex + | BRACKET_LEFT ASTERISK BRACKET_RIGHT # ExcludeExprCollectionWildcard + | PERIOD ASTERISK # ExcludeExprTupleWildcard + ; + fromClause : FROM tableReference; @@ -513,6 +530,7 @@ exprBagOp exprSelect : select=selectClause + exclude=excludeClause? from=fromClause let=letClause? where=whereClauseSelect? diff --git a/partiql-parser/src/main/antlr/PartiQLTokens.g4 b/partiql-parser/src/main/antlr/PartiQLTokens.g4 index dd0f56124..12d396d30 100644 --- a/partiql-parser/src/main/antlr/PartiQLTokens.g4 +++ b/partiql-parser/src/main/antlr/PartiQLTokens.g4 @@ -101,6 +101,7 @@ ESCAPE: 'ESCAPE'; EVERY: 'EVERY'; EXCEPT: 'EXCEPT'; EXCEPTION: 'EXCEPTION'; +EXCLUDE: 'EXCLUDE'; EXCLUDED: 'EXCLUDED'; EXEC: 'EXEC'; EXECUTE: 'EXECUTE'; diff --git a/partiql-parser/src/main/kotlin/org/partiql/parser/impl/PartiQLParserDefault.kt b/partiql-parser/src/main/kotlin/org/partiql/parser/impl/PartiQLParserDefault.kt index 0bcda31c4..bb0c33a28 100644 --- a/partiql-parser/src/main/kotlin/org/partiql/parser/impl/PartiQLParserDefault.kt +++ b/partiql-parser/src/main/kotlin/org/partiql/parser/impl/PartiQLParserDefault.kt @@ -35,6 +35,7 @@ import org.antlr.v4.runtime.tree.TerminalNode import org.partiql.ast.Ast import org.partiql.ast.AstNode import org.partiql.ast.DatetimeField +import org.partiql.ast.Exclude import org.partiql.ast.Expr import org.partiql.ast.From import org.partiql.ast.GraphMatch @@ -761,6 +762,7 @@ internal class PartiQLParserDefault : PartiQLParser { override fun visitSfwQuery(ctx: GeneratedParser.SfwQueryContext) = translate(ctx) { val select = visit(ctx.select) as Select val from = visitFromClause(ctx.from) + val exclude = visitOrNull(ctx.exclude) val let = visitOrNull(ctx.let) val where = visitOrNull(ctx.where) val groupBy = ctx.group?.let { visitGroupClause(it) } @@ -770,7 +772,7 @@ internal class PartiQLParserDefault : PartiQLParser { val orderBy = ctx.order?.let { visitOrderByClause(it) } val limit = visitOrNull(ctx.limit?.arg) val offset = visitOrNull(ctx.offset?.arg) - exprSFW(select, from, let, where, groupBy, having, setOp, orderBy, limit, offset) + exprSFW(select, exclude, from, let, where, groupBy, having, setOp, orderBy, limit, offset) } /** @@ -899,6 +901,49 @@ internal class PartiQLParserDefault : PartiQLParser { groupByKey(expr, alias) } + /** + * EXCLUDE CLAUSE + */ + override fun visitExcludeClause(ctx: GeneratedParser.ExcludeClauseContext) = translate(ctx) { + val excludeExprs = ctx.excludeExpr().map { expr -> + visitExcludeExpr(expr) + } + exclude(excludeExprs) + } + + override fun visitExcludeExpr(ctx: GeneratedParser.ExcludeExprContext) = translate(ctx) { + val root = visitSymbolPrimitive(ctx.symbolPrimitive()) + val steps = visitOrEmpty(ctx.excludeExprSteps()) + excludeExcludeExpr(root, steps) + } + + override fun visitExcludeExprTupleAttr(ctx: GeneratedParser.ExcludeExprTupleAttrContext) = translate(ctx) { + val identifier = visitSymbolPrimitive(ctx.symbolPrimitive()) + excludeStepExcludeTupleAttr(identifier) + } + + override fun visitExcludeExprCollectionIndex(ctx: GeneratedParser.ExcludeExprCollectionIndexContext) = translate(ctx) { + val index = ctx.index.text.toInt() + excludeStepExcludeCollectionIndex(index) + } + + override fun visitExcludeExprCollectionAttr(ctx: GeneratedParser.ExcludeExprCollectionAttrContext) = translate(ctx) { + val attr = ctx.attr.getStringValue() + val identifier = identifierSymbol( + attr, + Identifier.CaseSensitivity.SENSITIVE, + ) + excludeStepExcludeTupleAttr(identifier) + } + + override fun visitExcludeExprCollectionWildcard(ctx: org.partiql.parser.antlr.PartiQLParser.ExcludeExprCollectionWildcardContext) = translate(ctx) { + excludeStepExcludeCollectionWildcard() + } + + override fun visitExcludeExprTupleWildcard(ctx: org.partiql.parser.antlr.PartiQLParser.ExcludeExprTupleWildcardContext) = translate(ctx) { + excludeStepExcludeTupleWildcard() + } + /** * * BAG OPERATIONS diff --git a/partiql-plan/src/main/resources/partiql_plan.ion b/partiql-plan/src/main/resources/partiql_plan.ion index f29b034cd..26ee7b497 100644 --- a/partiql-plan/src/main/resources/partiql_plan.ion +++ b/partiql-plan/src/main/resources/partiql_plan.ion @@ -78,6 +78,25 @@ arg::[ } ] +// Exclude expr and step +exclude_expr::{ + root: string, + root_case: case, + steps: list::[exclude_step], +} + +exclude_step::[ + tuple_attr::{ + attr: string, + case: case, + }, + collection_index::{ + index: int, + }, + tuple_wildcard::{}, + collection_wildcard::{}, +] + branch::{ condition: rex, value: rex, @@ -149,6 +168,11 @@ rel::[ groups: list::[binding], strategy: [ FULL, PARTIAL ], }, + exclude::{ + common: common, + input: rel, + exprs: list::[exclude_expr], + } ] // Operators that return any value