From d11415264230f2650fd8d166898029423af681fa Mon Sep 17 00:00:00 2001 From: John Ed Quinn Date: Fri, 19 Apr 2024 16:01:42 -0700 Subject: [PATCH] Simplifies joins and fixes bugs --- .../org/partiql/eval/internal/Compiler.kt | 10 +- .../org/partiql/eval/internal/Environment.kt | 6 +- .../eval/internal/helpers/TypesUtility.kt | 80 +++++++++++ .../eval/internal/helpers/ValueUtility.kt | 20 +++ .../internal/operator/rel/RelJoinInner.kt | 63 +++++++- .../eval/internal/operator/rel/RelJoinLeft.kt | 18 --- .../operator/rel/RelJoinNestedLoop.kt | 100 ------------- .../internal/operator/rel/RelJoinOuterFull.kt | 136 ++++++++++++------ .../internal/operator/rel/RelJoinOuterLeft.kt | 83 +++++++++++ .../operator/rel/RelJoinOuterRight.kt | 84 +++++++++++ .../internal/operator/rel/RelJoinRight.kt | 21 --- .../eval/internal/operator/rex/ExprPathKey.kt | 4 + .../eval/internal/PartiQLEngineDefaultTest.kt | 125 +++++++++++++++- .../src/main/resources/partiql_plan.ion | 2 + 14 files changed, 554 insertions(+), 198 deletions(-) create mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/TypesUtility.kt create mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/ValueUtility.kt delete mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinLeft.kt delete mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinNestedLoop.kt create mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterLeft.kt create mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterRight.kt delete mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinRight.kt diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt index 6df4eaf590..d956b09c8c 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt @@ -7,9 +7,9 @@ import org.partiql.eval.internal.operator.rel.RelDistinct import org.partiql.eval.internal.operator.rel.RelExclude import org.partiql.eval.internal.operator.rel.RelFilter import org.partiql.eval.internal.operator.rel.RelJoinInner -import org.partiql.eval.internal.operator.rel.RelJoinLeft import org.partiql.eval.internal.operator.rel.RelJoinOuterFull -import org.partiql.eval.internal.operator.rel.RelJoinRight +import org.partiql.eval.internal.operator.rel.RelJoinOuterLeft +import org.partiql.eval.internal.operator.rel.RelJoinOuterRight import org.partiql.eval.internal.operator.rel.RelLimit import org.partiql.eval.internal.operator.rel.RelOffset import org.partiql.eval.internal.operator.rel.RelProject @@ -331,9 +331,9 @@ internal class Compiler( val condition = visitRex(node.rex, ctx) return when (node.type) { Rel.Op.Join.Type.INNER -> RelJoinInner(lhs, rhs, condition) - Rel.Op.Join.Type.LEFT -> RelJoinLeft(lhs, rhs, condition) - Rel.Op.Join.Type.RIGHT -> RelJoinRight(lhs, rhs, condition) - Rel.Op.Join.Type.FULL -> RelJoinOuterFull(lhs, rhs, condition) + Rel.Op.Join.Type.LEFT -> RelJoinOuterLeft(lhs, rhs, condition, rhsType = node.rhs.type) + Rel.Op.Join.Type.RIGHT -> RelJoinOuterRight(lhs, rhs, condition, lhsType = node.lhs.type) + Rel.Op.Join.Type.FULL -> RelJoinOuterFull(lhs, rhs, condition, lhsType = node.lhs.type, rhsType = node.rhs.type) } } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Environment.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Environment.kt index 4200766393..3385513c03 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Environment.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Environment.kt @@ -18,7 +18,11 @@ internal class Environment( @OptIn(PartiQLValueExperimental::class) operator fun get(index: Int): PartiQLValue { - return this.bindings[index] + try { + return this.bindings[index] + } catch (_: Throwable) { + throw IllegalStateException("Received error when searching for binding at index $index. Current bindings are: $this.") + } } @OptIn(PartiQLValueExperimental::class) diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/TypesUtility.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/TypesUtility.kt new file mode 100644 index 0000000000..5857f2eeec --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/TypesUtility.kt @@ -0,0 +1,80 @@ +package org.partiql.eval.internal.helpers + +import org.partiql.types.AnyOfType +import org.partiql.types.AnyType +import org.partiql.types.BagType +import org.partiql.types.BlobType +import org.partiql.types.BoolType +import org.partiql.types.ClobType +import org.partiql.types.DateType +import org.partiql.types.DecimalType +import org.partiql.types.FloatType +import org.partiql.types.GraphType +import org.partiql.types.IntType +import org.partiql.types.ListType +import org.partiql.types.MissingType +import org.partiql.types.NullType +import org.partiql.types.SexpType +import org.partiql.types.StaticType +import org.partiql.types.StringType +import org.partiql.types.StructType +import org.partiql.types.SymbolType +import org.partiql.types.TimeType +import org.partiql.types.TimestampType +import org.partiql.value.PartiQLValueExperimental +import org.partiql.value.PartiQLValueType + +internal object TypesUtility { + + @OptIn(PartiQLValueExperimental::class) + internal fun StaticType.toRuntimeType(): PartiQLValueType { + if (this is AnyOfType) { + // handle anyOf(null, T) cases + val t = types.filter { it !is NullType && it !is MissingType } + return if (t.size != 1) { + PartiQLValueType.ANY + } else { + t.first().asRuntimeType() + } + } + return this.asRuntimeType() + } + + @OptIn(PartiQLValueExperimental::class) + private fun StaticType.asRuntimeType(): PartiQLValueType = when (this) { + is AnyOfType -> PartiQLValueType.ANY + is AnyType -> PartiQLValueType.ANY + is BlobType -> PartiQLValueType.BLOB + is BoolType -> PartiQLValueType.BOOL + is ClobType -> PartiQLValueType.CLOB + is BagType -> PartiQLValueType.BAG + is ListType -> PartiQLValueType.LIST + is SexpType -> PartiQLValueType.SEXP + is DateType -> PartiQLValueType.DATE + // TODO: Run time decimal type does not model precision scale constraint yet + // despite that we match to Decimal vs Decimal_ARBITRARY (PVT) here + // but when mapping it back to Static Type, (i.e, mapping function return type to Value Type) + // we can only map to Unconstrained decimal (Static Type) + is DecimalType -> { + when (this.precisionScaleConstraint) { + is DecimalType.PrecisionScaleConstraint.Constrained -> PartiQLValueType.DECIMAL + DecimalType.PrecisionScaleConstraint.Unconstrained -> PartiQLValueType.DECIMAL_ARBITRARY + } + } + is FloatType -> PartiQLValueType.FLOAT64 + is GraphType -> error("Graph type missing from runtime types") + is IntType -> when (this.rangeConstraint) { + IntType.IntRangeConstraint.SHORT -> PartiQLValueType.INT16 + IntType.IntRangeConstraint.INT4 -> PartiQLValueType.INT32 + IntType.IntRangeConstraint.LONG -> PartiQLValueType.INT64 + IntType.IntRangeConstraint.UNCONSTRAINED -> PartiQLValueType.INT + } + MissingType -> PartiQLValueType.MISSING + is NullType -> PartiQLValueType.NULL + is StringType -> PartiQLValueType.STRING + is StructType -> PartiQLValueType.STRUCT + is SymbolType -> PartiQLValueType.SYMBOL + is TimeType -> PartiQLValueType.TIME + is TimestampType -> PartiQLValueType.TIMESTAMP + } +} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/ValueUtility.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/ValueUtility.kt new file mode 100644 index 0000000000..b1682ad93e --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/ValueUtility.kt @@ -0,0 +1,20 @@ +package org.partiql.eval.internal.helpers + +import org.partiql.value.BoolValue +import org.partiql.value.PartiQLValue +import org.partiql.value.PartiQLValueExperimental + +/** + * Holds helper functions for [PartiQLValue]. + */ +internal object ValueUtility { + + /** + * @return whether the value is a boolean and the value itself is not-null and true. + */ + @OptIn(PartiQLValueExperimental::class) + @JvmStatic + fun PartiQLValue.isTrue(): Boolean { + return this is BoolValue && this.value == true + } +} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinInner.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinInner.kt index 265e78a549..1a71d2c53f 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinInner.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinInner.kt @@ -1,17 +1,66 @@ package org.partiql.eval.internal.operator.rel +import org.partiql.eval.internal.Environment import org.partiql.eval.internal.Record +import org.partiql.eval.internal.helpers.ValueUtility.isTrue import org.partiql.eval.internal.operator.Operator +import org.partiql.value.PartiQLValueExperimental internal class RelJoinInner( - override val lhs: Operator.Relation, - override val rhs: Operator.Relation, - override val condition: Operator.Expr, -) : RelJoinNestedLoop() { - override fun join(condition: Boolean, lhs: Record, rhs: Record): Record? { - return when (condition) { - true -> lhs + rhs + private val lhs: Operator.Relation, + private val rhs: Operator.Relation, + private val condition: Operator.Expr, +) : RelPeeking() { + + private lateinit var env: Environment + private lateinit var iterator: Iterator + + override fun open(env: Environment) { + this.env = env + lhs.open(env) + iterator = implementation() + super.open(env) + } + + override fun peek(): Record? { + return when (iterator.hasNext()) { + true -> iterator.next() false -> null } } + + override fun close() { + lhs.close() + rhs.close() + iterator = emptyList().iterator() + super.close() + } + + /** + * INNER JOIN (LATERAL) + * + * Algorithm: + * ``` + * for lhsRecord in lhs: + * for rhsRecord in rhs(lhsRecord): + * if (condition matches): + * conditionMatched = true + * yield(lhsRecord + rhsRecord) + * ``` + * + * Development Note: The non-lateral version wouldn't need to push to the current environment. + */ + @OptIn(PartiQLValueExperimental::class) + private fun implementation() = iterator { + for (lhsRecord in lhs) { + rhs.open(env.push(lhsRecord)) + for (rhsRecord in rhs) { + val input = lhsRecord + rhsRecord + val result = condition.eval(env.push(input)) + if (result.isTrue()) { + yield(lhsRecord + rhsRecord) + } + } + } + } } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinLeft.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinLeft.kt deleted file mode 100644 index eae51e9a31..0000000000 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinLeft.kt +++ /dev/null @@ -1,18 +0,0 @@ -package org.partiql.eval.internal.operator.rel - -import org.partiql.eval.internal.Record -import org.partiql.eval.internal.operator.Operator - -internal class RelJoinLeft( - override val lhs: Operator.Relation, - override val rhs: Operator.Relation, - override val condition: Operator.Expr, -) : RelJoinNestedLoop() { - - override fun join(condition: Boolean, lhs: Record, rhs: Record): Record { - if (condition.not()) { - rhs.padNull() - } - return lhs + rhs - } -} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinNestedLoop.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinNestedLoop.kt deleted file mode 100644 index ed761c10d2..0000000000 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinNestedLoop.kt +++ /dev/null @@ -1,100 +0,0 @@ -package org.partiql.eval.internal.operator.rel - -import org.partiql.eval.internal.Environment -import org.partiql.eval.internal.Record -import org.partiql.eval.internal.operator.Operator -import org.partiql.value.BoolValue -import org.partiql.value.PartiQLValue -import org.partiql.value.PartiQLValueExperimental -import org.partiql.value.StructValue -import org.partiql.value.nullValue -import org.partiql.value.structValue - -internal abstract class RelJoinNestedLoop : RelPeeking() { - - abstract val lhs: Operator.Relation - abstract val rhs: Operator.Relation - abstract val condition: Operator.Expr - - private var lhsRecord: Record? = null - private lateinit var env: Environment - - override fun open(env: Environment) { - this.env = env - lhs.open(env) - if (lhs.hasNext().not()) { - return - } - lhsRecord = lhs.next() - rhs.open(env.push(lhsRecord!!)) - super.open(env) - } - - abstract fun join(condition: Boolean, lhs: Record, rhs: Record): Record? - - @OptIn(PartiQLValueExperimental::class) - override fun peek(): Record? { - if (lhsRecord == null) { - return null - } - var rhsRecord = when (rhs.hasNext()) { - true -> rhs.next() - false -> null - } - var toReturn: Record? = null - do { - // Acquire LHS and RHS Records - if (rhsRecord == null) { - rhs.close() - if (lhs.hasNext().not()) { - return null - } - lhsRecord = lhs.next() - rhs.open(env.push(lhsRecord!!)) - rhsRecord = when (rhs.hasNext()) { - true -> rhs.next() - false -> null - } - } - // Return Joined Record - if (rhsRecord != null && lhsRecord != null) { - val input = lhsRecord!! + rhsRecord - val result = condition.eval(env.push(input)) - toReturn = join(result.isTrue(), lhsRecord!!, rhsRecord) - } - // Move the pointer to the next row for the RHS - if (toReturn == null) rhsRecord = if (rhs.hasNext()) rhs.next() else null - } - while (toReturn == null) - return toReturn - } - - override fun close() { - lhs.close() - rhs.close() - super.close() - } - - @OptIn(PartiQLValueExperimental::class) - private fun PartiQLValue.isTrue(): Boolean { - return this is BoolValue && this.value == true - } - - @OptIn(PartiQLValueExperimental::class) - internal fun Record.padNull() { - this.values.indices.forEach { index -> - this.values[index] = values[index].padNull() - } - } - - @OptIn(PartiQLValueExperimental::class) - private fun PartiQLValue.padNull(): PartiQLValue { - return when (this) { - is StructValue<*> -> { - val newFields = this.entries.map { it.first to nullValue() } - structValue(newFields) - } - else -> nullValue() - } - } -} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterFull.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterFull.kt index 546c161715..d0e9bf9bd0 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterFull.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterFull.kt @@ -1,61 +1,107 @@ package org.partiql.eval.internal.operator.rel +import org.partiql.eval.internal.Environment import org.partiql.eval.internal.Record +import org.partiql.eval.internal.helpers.TypesUtility.toRuntimeType +import org.partiql.eval.internal.helpers.ValueUtility.isTrue +import org.partiql.eval.internal.helpers.toNull import org.partiql.eval.internal.operator.Operator +import org.partiql.plan.Rel +import org.partiql.value.PartiQLValueExperimental -/** - * Here's a simple implementation of FULL OUTER JOIN. The idea is fairly straightforward: - * Iterate through LHS. For each iteration of the LHS, iterate through RHS. Now, check the condition. - * - If the condition passes, return the merged record (equivalent to result of INNER JOIN) - * - If the condition does not pass, we need a way to return two records (one where the LHS is padded with nulls, and - * one where the RHS is padded with nulls). How we do this: - * - We maintain the [previousLhs] and [previousRhs]. If they are null, we then compute the next LHS and RHS. We - * store their values in-memory. Then we return a merged Record where the LHS is padded and the RHS is not (equivalent - * to result of RIGHT OUTER JOIN). - * - If they aren't null, then we pad the RHS with NULLS (we assume we've already padded the LHS) and return (equivalent - * to result of LEFT OUTER JOIN). We also make sure [previousLhs] and [previousRhs] are now null. - * - * Performance Analysis: Assume that [lhs] has size M and [rhs] has size N. - * - Time: O(M * N) - * - Space: O(1) - */ internal class RelJoinOuterFull( - override val lhs: Operator.Relation, - override val rhs: Operator.Relation, - override val condition: Operator.Expr, -) : RelJoinNestedLoop() { - - private var previousLhs: Record? = null - private var previousRhs: Record? = null - - override fun next(): Record { - if (previousLhs != null && previousRhs != null) { - previousRhs!!.padNull() - val newRecord = previousLhs!! + previousRhs!! - previousLhs = null - previousRhs = null - return newRecord + private val lhs: Operator.Relation, + private val rhs: Operator.Relation, + private val condition: Operator.Expr, + lhsType: Rel.Type, + rhsType: Rel.Type +) : RelPeeking() { + + @OptIn(PartiQLValueExperimental::class) + private val lhsPadded = Record( + Array(rhsType.schema.size) { lhsType.schema[it].type.toRuntimeType().toNull().invoke() } + ) + + @OptIn(PartiQLValueExperimental::class) + private val rhsPadded = Record( + Array(rhsType.schema.size) { rhsType.schema[it].type.toRuntimeType().toNull().invoke() } + ) + + private lateinit var env: Environment + private lateinit var iterator: Iterator + + override fun open(env: Environment) { + this.env = env + lhs.open(env) + iterator = implementation() + super.open(env) + } + + override fun peek(): Record? { + return when (iterator.hasNext()) { + true -> iterator.next() + false -> null } - return super.next() + } + + override fun close() { + lhs.close() + rhs.close() + iterator = emptyList().iterator() + super.close() } /** - * Specifically, for FULL OUTER JOIN, when the JOIN Condition ([condition]) is TRUE, we need to return the - * rows merged (without modification). When the JOIN Condition ([condition]) is FALSE, we need to return - * the LHS padded (and merged with RHS not padded) and the RHS padded (merged with the LHS not padded). + * FULL OUTER JOIN (CANNOT BE LATERAL) + * + * Algorithm: + * ``` + * for lhsRecord, lhsIndex in lhs_sorted: + * for rhsRecord, rhsIndex in rhs_sorted: + * if (condition matches): + * lhsMatches[lhsIndex] = true + * rhsMatches[rhsIndex] = true + * yield(lhsRecord + rhsRecord) + * for lhsRecord, lhsIndex in lhs_sorted: + * if lhsMatches[lhsIndex] = false: + * yield(lhsRecord, null) + * for rhsRecord, rhsIndex in rhs_sorted: + * if rhsMatches[rhsIndex] = false: + * yield(null, rhsRecord) + * ``` + * + * Note: The LHS and RHS must be sorted. TODO: We need to add sorting to the LHS and RHS */ - override fun join(condition: Boolean, lhs: Record, rhs: Record): Record { - when (condition) { - true -> { - previousLhs = null - previousRhs = null + @OptIn(PartiQLValueExperimental::class) + private fun implementation() = iterator { + val lhsMatches = mutableSetOf() + val rhsMatches = mutableSetOf() + for ((lhsIndex, lhsRecord) in lhs.withIndex()) { + rhs.open(env) + for ((rhsIndex, rhsRecord) in rhs.withIndex()) { + val input = lhsRecord + rhsRecord + val result = condition.eval(env.push(input)) + if (result.isTrue()) { + lhsMatches.add(lhsIndex) + rhsMatches.add(rhsIndex) + yield(lhsRecord + rhsRecord) + } } - false -> { - previousLhs = lhs.copy() - previousRhs = rhs.copy() - lhs.padNull() + rhs.close() + } + lhs.close() + lhs.open(env) + for ((lhsIndex, lhsRecord) in lhs.withIndex()) { + if (!lhsMatches.contains(lhsIndex)) { + yield(lhsRecord + rhsPadded) + } + } + lhs.close() + rhs.open(env) + for ((rhsIndex, rhsRecord) in rhs.withIndex()) { + if (!rhsMatches.contains(rhsIndex)) { + yield(lhsPadded + rhsRecord) } } - return lhs + rhs } } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterLeft.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterLeft.kt new file mode 100644 index 0000000000..a160914d7d --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterLeft.kt @@ -0,0 +1,83 @@ +package org.partiql.eval.internal.operator.rel + +import org.partiql.eval.internal.Environment +import org.partiql.eval.internal.Record +import org.partiql.eval.internal.helpers.TypesUtility.toRuntimeType +import org.partiql.eval.internal.helpers.ValueUtility.isTrue +import org.partiql.eval.internal.helpers.toNull +import org.partiql.eval.internal.operator.Operator +import org.partiql.plan.Rel +import org.partiql.value.PartiQLValueExperimental + +internal class RelJoinOuterLeft( + private val lhs: Operator.Relation, + private val rhs: Operator.Relation, + private val condition: Operator.Expr, + rhsType: Rel.Type +) : RelPeeking() { + + @OptIn(PartiQLValueExperimental::class) + private val rhsPadded = Record( + Array(rhsType.schema.size) { rhsType.schema[it].type.toRuntimeType().toNull().invoke() } + ) + + private lateinit var env: Environment + private lateinit var iterator: Iterator + + override fun open(env: Environment) { + this.env = env + lhs.open(env) + iterator = implementation() + super.open(env) + } + + override fun peek(): Record? { + return when (iterator.hasNext()) { + true -> iterator.next() + false -> null + } + } + + override fun close() { + lhs.close() + rhs.close() + iterator = emptyList().iterator() + super.close() + } + + /** + * LEFT OUTER JOIN (LATERAL) + * + * Algorithm: + * ``` + * for lhsRecord in lhs: + * for rhsRecord in rhs(lhsRecord): + * if (condition matches): + * conditionMatched = true + * yield(lhsRecord + rhsRecord) + * if (!conditionMatched): + * yield(lhsRecord + NULL_RECORD) + * ``` + * + * Development Note: The non-lateral version wouldn't need to push to the current environment. + */ + @OptIn(PartiQLValueExperimental::class) + private fun implementation() = iterator { + for (lhsRecord in lhs) { + var lhsMatched = false + rhs.open(env.push(lhsRecord)) + for (rhsRecord in rhs) { + val input = lhsRecord + rhsRecord + val result = condition.eval(env.push(input)) + if (result.isTrue()) { + lhsMatched = true + yield(lhsRecord + rhsRecord) + } + } + rhs.close() + if (!lhsMatched) { + yield(lhsRecord + rhsPadded) + } + } + } +} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterRight.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterRight.kt new file mode 100644 index 0000000000..13e41f03e4 --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinOuterRight.kt @@ -0,0 +1,84 @@ +package org.partiql.eval.internal.operator.rel + +import org.partiql.eval.internal.Environment +import org.partiql.eval.internal.Record +import org.partiql.eval.internal.helpers.TypesUtility.toRuntimeType +import org.partiql.eval.internal.helpers.ValueUtility.isTrue +import org.partiql.eval.internal.helpers.toNull +import org.partiql.eval.internal.operator.Operator +import org.partiql.plan.Rel +import org.partiql.value.PartiQLValueExperimental + +internal class RelJoinOuterRight( + private val lhs: Operator.Relation, + private val rhs: Operator.Relation, + private val condition: Operator.Expr, + lhsType: Rel.Type +) : RelPeeking() { + + @OptIn(PartiQLValueExperimental::class) + private val lhsPadded = Record( + Array(lhsType.schema.size) { lhsType.schema[it].type.toRuntimeType().toNull().invoke() } + ) + + private lateinit var env: Environment + private lateinit var iterator: Iterator + + override fun open(env: Environment) { + this.env = env + rhs.open(env) + iterator = implementation() + super.open(env) + } + + override fun peek(): Record? { + return when (iterator.hasNext()) { + true -> iterator.next() + false -> null + } + } + + override fun close() { + lhs.close() + rhs.close() + iterator = emptyList().iterator() + super.close() + } + + /** + * RIGHT OUTER JOIN (LATERAL) + * + * Algorithm: + * ``` + * for rhsRecord in rhs: + * for lhsRecord in lhs(rhsRecord): + * if (condition matches): + * conditionMatched = true + * yield(lhsRecord + rhsRecord) + * if (!conditionMatched): + * yield(NULL_RECORD + rhsRecord) + * ``` + * + * Another Note: For some databases, RIGHT OUTER JOIN cannot be used with LATERAL. This should be taken care of + * at planning time. TODO: Should we support LATERAL here? Can we deal with this at planning time? + */ + @OptIn(PartiQLValueExperimental::class) + private fun implementation() = iterator { + for (rhsRecord in rhs) { + var rhsMatched = false + lhs.open(env.push(rhsRecord)) + for (lhsRecord in lhs) { + val input = lhsRecord + rhsRecord + val result = condition.eval(env.push(input)) + if (result.isTrue()) { + rhsMatched = true + yield(lhsRecord + rhsRecord) + } + } + lhs.close() + if (!rhsMatched) { + yield(lhsPadded + rhsRecord) + } + } + } +} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinRight.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinRight.kt deleted file mode 100644 index 4c020f93f8..0000000000 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinRight.kt +++ /dev/null @@ -1,21 +0,0 @@ -package org.partiql.eval.internal.operator.rel - -import org.partiql.eval.internal.Record -import org.partiql.eval.internal.operator.Operator - -internal class RelJoinRight( - lhs: Operator.Relation, - rhs: Operator.Relation, - override val condition: Operator.Expr, -) : RelJoinNestedLoop() { - - override val lhs: Operator.Relation = rhs - override val rhs: Operator.Relation = lhs - - override fun join(condition: Boolean, lhs: Record, rhs: Record): Record { - if (condition.not()) { - lhs.padNull() - } - return lhs + rhs - } -} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprPathKey.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprPathKey.kt index 34000c7fb5..4927096419 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprPathKey.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprPathKey.kt @@ -8,6 +8,7 @@ import org.partiql.value.PartiQLValueExperimental import org.partiql.value.StringValue import org.partiql.value.StructValue import org.partiql.value.check +import org.partiql.value.nullValue internal class ExprPathKey( @JvmField val root: Operator.Expr, @@ -19,6 +20,9 @@ internal class ExprPathKey( val rootEvaluated = root.eval(env).check>() val keyEvaluated = key.eval(env).check() val keyString = keyEvaluated.value ?: error("String value was null") + if (rootEvaluated.isNull || keyEvaluated.isNull) { + return nullValue() + } return rootEvaluated[keyString] ?: throw TypeCheckException() } } diff --git a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt index 6de65d6937..8c0cd1e4b2 100644 --- a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt +++ b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt @@ -65,6 +65,11 @@ class PartiQLEngineDefaultTest { @Execution(ExecutionMode.CONCURRENT) fun aggregationTests(tc: SuccessTestCase) = tc.assert() + @ParameterizedTest + @MethodSource("joinTestCases") + @Execution(ExecutionMode.CONCURRENT) + fun joinTests(tc: SuccessTestCase) = tc.assert() + @ParameterizedTest @MethodSource("globalsTestCases") @Execution(ExecutionMode.CONCURRENT) @@ -195,6 +200,123 @@ class PartiQLEngineDefaultTest { ), ) + @JvmStatic + fun joinTestCases() = listOf( + // LEFT OUTER JOIN -- Easy + SuccessTestCase( + input = """ + SELECT VALUE [lhs, rhs] + FROM << 0, 1, 2 >> lhs + LEFT OUTER JOIN << 0, 2, 3 >> rhs + ON lhs = rhs + """.trimIndent(), + expected = bagValue( + listValue(int32Value(0), int32Value(0)), + listValue(int32Value(1), int32Value(null)), + listValue(int32Value(2), int32Value(2)), + ) + ), + // LEFT OUTER JOIN -- RHS Empty + SuccessTestCase( + input = """ + SELECT VALUE [lhs, rhs] + FROM + << 0, 1, 2 >> lhs + LEFT OUTER JOIN ( + SELECT VALUE n + FROM << 0, 2, 3 >> AS n + WHERE n > 100 + ) rhs + ON lhs = rhs + """.trimIndent(), + expected = bagValue( + listValue(int32Value(0), int32Value(null)), + listValue(int32Value(1), int32Value(null)), + listValue(int32Value(2), int32Value(null)), + ) + ), + // LEFT OUTER JOIN -- LHS Empty + SuccessTestCase( + input = """ + SELECT VALUE [lhs, rhs] + FROM <<>> lhs + LEFT OUTER JOIN << 0, 2, 3>> rhs + ON lhs = rhs + """.trimIndent(), + expected = bagValue() + ), + // LEFT OUTER JOIN -- No Matches + SuccessTestCase( + input = """ + SELECT VALUE [lhs, rhs] + FROM << 0, 1, 2 >> lhs + LEFT OUTER JOIN << 3, 4, 5 >> rhs + ON lhs = rhs + """.trimIndent(), + expected = bagValue( + listValue(int32Value(0), int32Value(null)), + listValue(int32Value(1), int32Value(null)), + listValue(int32Value(2), int32Value(null)), + ) + ), + // RIGHT OUTER JOIN -- Easy + SuccessTestCase( + input = """ + SELECT VALUE [lhs, rhs] + FROM << 0, 1, 2 >> lhs + RIGHT OUTER JOIN << 0, 2, 3 >> rhs + ON lhs = rhs + """.trimIndent(), + expected = bagValue( + listValue(int32Value(0), int32Value(0)), + listValue(int32Value(2), int32Value(2)), + listValue(int32Value(null), int32Value(3)), + ) + ), + // RIGHT OUTER JOIN -- RHS Empty + SuccessTestCase( + input = """ + SELECT VALUE [lhs, rhs] + FROM << 0, 1, 2 >> lhs + RIGHT OUTER JOIN <<>> rhs + ON lhs = rhs + """.trimIndent(), + expected = bagValue() + ), + // RIGHT OUTER JOIN -- LHS Empty + SuccessTestCase( + input = """ + SELECT VALUE [lhs, rhs] + FROM ( + SELECT VALUE n + FROM << 0, 1, 2 >> AS n + WHERE n > 100 + ) lhs RIGHT OUTER JOIN + << 0, 2, 3>> rhs + ON lhs = rhs + """.trimIndent(), + expected = bagValue( + listValue(int32Value(null), int32Value(0)), + listValue(int32Value(null), int32Value(2)), + listValue(int32Value(null), int32Value(3)), + ) + ), + // RIGHT OUTER JOIN -- No Matches + SuccessTestCase( + input = """ + SELECT VALUE [lhs, rhs] + FROM << 0, 1, 2 >> lhs + RIGHT OUTER JOIN << 3, 4, 5 >> rhs + ON lhs = rhs + """.trimIndent(), + expected = bagValue( + listValue(int32Value(null), int32Value(3)), + listValue(int32Value(null), int32Value(4)), + listValue(int32Value(null), int32Value(5)), + ) + ), + ) + @JvmStatic fun subqueryTestCases() = listOf( SuccessTestCase( @@ -598,7 +720,8 @@ class PartiQLEngineDefaultTest { ), SuccessTestCase( input = "SELECT t.a, s.b FROM << { 'a': 1 } >> t LEFT JOIN << { 'b': 2 } >> s ON false;", - expected = bagValue(structValue("a" to int32Value(1), "b" to nullValue())) + expected = bagValue(structValue("a" to int32Value(1), "b" to nullValue())), + mode = PartiQLEngine.Mode.STRICT ), SuccessTestCase( input = "SELECT t.a, s.b FROM << { 'a': 1 } >> t FULL OUTER JOIN << { 'b': 2 } >> s ON false;", diff --git a/partiql-plan/src/main/resources/partiql_plan.ion b/partiql-plan/src/main/resources/partiql_plan.ion index 0c272060b8..63b7b4f3a6 100644 --- a/partiql-plan/src/main/resources/partiql_plan.ion +++ b/partiql-plan/src/main/resources/partiql_plan.ion @@ -291,6 +291,8 @@ rel::{ projections: list::[rex], }, + // TODO: Specify that this is a LATERAL JOIN. Create a separate JOIN. Also, determine the allowable types of JOIN. + // For context: Oracle SQL doesn't allow ... FULL OUTER JOIN LATERAL ... or ... RIGHT OUTER JOIN LATERAL ... join::{ lhs: rel, rhs: rel,