Skip to content

Commit

Permalink
refactor(interactive): Support Logical Operator within in Index Pre…
Browse files Browse the repository at this point in the history 
…dicate (#3398)

<!--
Thanks for your contribution! please review
https://github.com/alibaba/GraphScope/blob/main/CONTRIBUTING.md before
opening an issue.
-->

## What do these changes do?
as titled.

<!-- Please give a short brief about these changes. -->

## Related issue number

<!-- Are there any issues opened that will be resolved by merging this
change? -->

Fixes

---------

Co-authored-by: bingqing.lbq <[email protected]>
  • Loading branch information
@shirly121 @BingqingLyu
shirly121 and BingqingLyu authored Dec 6, 2023
1 parent ac31a91 commit c19808d
Show file tree
Hide file tree
Showing 11 changed files with 407 additions and 197 deletions.
9 changes: 8 additions & 1 deletion docs/interactive_engine/tinkerpop/faq.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,14 @@ For example, in the LDBC schema, we define the property ID as the primary key fo
```scss
g.V().hasLabel('PERSON').has('id', propertyIdValue)
```
Where 'id' is the property ID, and 'propertyIdValue' is the value of the property key. By directly using the primary key index, query performance can be significantly improved, avoiding full table scans and property value filtering, thereby optimizing query performance.
Where 'id' is the property ID, and 'propertyIdValue' is the value of the property key.

Moreover, we support the `within` operator to query multiple values of the same property key, which can also be optimized by the primary key index. For example:
```scss
g.V().hasLabel('PERSON').has('id', within(propertyIdValue1, propertyIdValue2))
```

By directly using the primary key index, query performance can be significantly improved, avoiding full table scans and property value filtering, thereby optimizing query performance.

## How to use subgraph in GIE Gremlin ?

Expand Down
116 changes: 105 additions & 11 deletions interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,18 @@
import com.alibaba.graphscope.gaia.proto.GraphAlgebra;
import com.alibaba.graphscope.gaia.proto.OuterExpression;
import com.alibaba.graphscope.gremlin.Utils;
import com.google.common.base.Preconditions;
import com.sun.jna.Pointer;
import com.sun.jna.ptr.IntByReference;

import org.javatuples.Pair;

import java.io.Closeable;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;

// represent ir plan as a chain of operators
public class IrPlan implements Closeable {
Expand Down Expand Up @@ -78,22 +82,17 @@ public Pointer apply(InterOpBase baseOp) {
// set index predicate
Optional<OpArg> ids = op.getIds();
if (ids.isPresent()) {
Pointer idsPredicate = irCoreLib.initIndexPredicate();
List<FfiConst.ByValue> ffiIds = (List<FfiConst.ByValue>) ids.get().applyArg();
for (int i = 0; i < ffiIds.size(); ++i) {
FfiResult e2 =
irCoreLib.orEquivPredicate(
idsPredicate, ArgUtils.asKey(ArgUtils.ID), ffiIds.get(i));
if (e2.code != ResultCode.Success) {
if (!ffiIds.isEmpty()) {
FfiResult.ByValue res =
irCoreLib.addIndexPredicatePb(scan, ffiIndexPredicate(ffiIds));
if (res.code != ResultCode.Success) {
throw new InterOpIllegalArgException(
baseOp.getClass(),
"ids",
"orEquivPredicate returns " + e2.getMsg());
"addIndexPredicatePb returns " + res.getMsg());
}
}
if (!ffiIds.isEmpty()) {
irCoreLib.addScanIndexPredicate(scan, idsPredicate);
}
}

if (op.isCountOnly()) {
Expand All @@ -107,6 +106,101 @@ public Pointer apply(InterOpBase baseOp) {
}
return scan;
}

private FfiPbPointer.ByValue ffiIndexPredicate(List<FfiConst.ByValue> ffiIds) {
GraphAlgebra.IndexPredicate.Triplet.Builder tripletBuilder =
GraphAlgebra.IndexPredicate.Triplet.newBuilder()
.setKey(
OuterExpression.Property.newBuilder()
.setId(OuterExpression.IdKey.newBuilder().build())
.build());
if (ffiIds.size() == 1) {
tripletBuilder.setCmp(OuterExpression.Logical.EQ);
} else {
tripletBuilder.setCmp(OuterExpression.Logical.WITHIN);
}
tripletBuilder.setConst(protoValue(ffiIds));
return new FfiPbPointer.ByValue(
GraphAlgebra.IndexPredicate.newBuilder()
.addOrPredicates(
GraphAlgebra.IndexPredicate.AndPredicate.newBuilder()
.addPredicates(tripletBuilder))
.build()
.toByteArray());
}

private Common.Value protoValue(List<FfiConst.ByValue> ffiIds) {
Preconditions.checkArgument(!ffiIds.isEmpty(), "ffiIds should not be empty");
FfiConst.ByValue constVal = ffiIds.get(0);
if (ffiIds.size() == 1) {
switch (constVal.dataType) {
case I32:
return Common.Value.newBuilder().setI32(constVal.int32).build();
case I64:
return Common.Value.newBuilder().setI64(constVal.int64).build();
case Str:
return Common.Value.newBuilder().setStr(constVal.cstr).build();
case Boolean:
return Common.Value.newBuilder().setBoolean(constVal.bool).build();
case F64:
return Common.Value.newBuilder().setF64(constVal.float64).build();
case Unknown:
default:
throw new IllegalArgumentException(
"cannot convert "
+ constVal.dataType
+ " to basic type in proto");
}
} else {
switch (constVal.dataType) {
case I32:
return Common.Value.newBuilder()
.setI32Array(
Common.I32Array.newBuilder()
.addAllItem(
ffiIds.stream()
.map(k -> k.int32)
.collect(Collectors.toList()))
.build())
.build();
case I64:
return Common.Value.newBuilder()
.setI64Array(
Common.I64Array.newBuilder()
.addAllItem(
ffiIds.stream()
.map(k -> k.int64)
.collect(Collectors.toList()))
.build())
.build();
case Str:
return Common.Value.newBuilder()
.setStrArray(
Common.StringArray.newBuilder()
.addAllItem(
ffiIds.stream()
.map(k -> k.cstr)
.collect(Collectors.toList()))
.build())
.build();
case F64:
return Common.Value.newBuilder()
.setF64Array(
Common.DoubleArray.newBuilder()
.addAllItem(
ffiIds.stream()
.map(k -> k.float64)
.collect(Collectors.toList()))
.build())
.build();
default:
throw new IllegalArgumentException(
"cannot convert array of "
+ constVal.dataType
+ " to array type in proto");
}
}
}
},
SELECT_OP {
@Override
Expand Down
97 changes: 11 additions & 86 deletions ...ompiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,17 @@
import com.alibaba.graphscope.common.ir.rel.type.group.GraphGroupKeys;
import com.alibaba.graphscope.common.ir.rel.type.order.GraphFieldCollation;
import com.alibaba.graphscope.common.ir.rex.RexGraphVariable;
import com.alibaba.graphscope.common.ir.runtime.proto.RexToIndexPbConverter;
import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter;
import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode;
import com.alibaba.graphscope.common.ir.tools.AliasInference;
import com.alibaba.graphscope.common.ir.tools.GraphPlanner;
import com.alibaba.graphscope.common.ir.tools.config.GraphOpt;
import com.alibaba.graphscope.common.ir.type.GraphLabelType;
import com.alibaba.graphscope.common.ir.type.GraphNameOrId;
import com.alibaba.graphscope.common.ir.type.GraphProperty;
import com.alibaba.graphscope.common.ir.type.GraphSchemaType;
import com.alibaba.graphscope.common.jna.IrCoreLibrary;
import com.alibaba.graphscope.common.jna.type.*;
import com.alibaba.graphscope.gaia.proto.GraphAlgebra;
import com.alibaba.graphscope.gaia.proto.OuterExpression;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
Expand All @@ -55,7 +55,6 @@
import org.apache.calcite.rex.*;
import org.apache.calcite.sql.SqlKind;
import org.apache.commons.lang3.ObjectUtils;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -81,9 +80,10 @@ public RelToFfiConverter(boolean isColumnId, Configs configs) {
@Override
public RelNode visit(GraphLogicalSource source) {
Pointer ptrScan = LIB.initScanOperator(Utils.ffiScanOpt(source.getOpt()));
Pointer ptrIndex = ffiIndexPredicates(source);
if (ptrIndex != null) {
checkFfiResult(LIB.addScanIndexPredicate(ptrScan, ptrIndex));
if (source.getUniqueKeyFilters() != null) {
checkFfiResult(
LIB.addIndexPredicatePb(
ptrScan, ffiIndexPredicates(source.getUniqueKeyFilters())));
}
checkFfiResult(LIB.setScanParams(ptrScan, ffiQueryParams(source)));
if (source.getAliasId() != AliasInference.DEFAULT_ID) {
Expand Down Expand Up @@ -505,86 +505,11 @@ private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) {
return params;
}

private @Nullable Pointer ffiIndexPredicates(GraphLogicalSource source) {
RexNode uniqueKeyFilters = source.getUniqueKeyFilters();
if (uniqueKeyFilters == null) return null;
// 'within' operator in index predicate is unsupported in ir core, here just expand it to
// 'or'
// i.e. '~id within [1, 2]' -> '~id == 1 or ~id == 2'
RexNode expandSearch = RexUtil.expandSearch(this.rexBuilder, null, uniqueKeyFilters);
List<RexNode> disjunctions = RelOptUtil.disjunctions(expandSearch);
Pointer ptrIndex = LIB.initIndexPredicate();
for (RexNode disjunction : disjunctions) {
if (disjunction instanceof RexCall) {
RexCall rexCall = (RexCall) disjunction;
switch (rexCall.getOperator().getKind()) {
case EQUALS:
RexNode left = rexCall.getOperands().get(0);
RexNode right = rexCall.getOperands().get(1);
if (left instanceof RexGraphVariable
&& (right instanceof RexLiteral
|| right instanceof RexDynamicParam)) {
LIB.orEquivPredicate(
ptrIndex,
getFfiProperty(((RexGraphVariable) left).getProperty()),
getFfiConst(right));
break;
} else if (right instanceof RexGraphVariable
&& (left instanceof RexLiteral
|| left instanceof RexDynamicParam)) {
LIB.orEquivPredicate(
ptrIndex,
getFfiProperty(((RexGraphVariable) right).getProperty()),
getFfiConst(left));
break;
}
default:
throw new IllegalArgumentException(
"can not convert unique key filter pattern="
+ rexCall
+ " to ir core index predicate");
}
} else {
throw new IllegalArgumentException(
"invalid unique key filter pattern=" + disjunction);
}
}
return ptrIndex;
}

private FfiProperty.ByValue getFfiProperty(GraphProperty property) {
Preconditions.checkArgument(property != null, "unique key should not be null");
FfiProperty.ByValue ffiProperty = new FfiProperty.ByValue();
switch (property.getOpt()) {
case ID:
ffiProperty.opt = FfiPropertyOpt.Id;
break;
case KEY:
ffiProperty.opt = FfiPropertyOpt.Key;
ffiProperty.key = getFfiNameOrId(property.getKey());
break;
default:
throw new IllegalArgumentException(
"can not convert property=" + property + " to ffi property");
}
return ffiProperty;
}

private FfiNameOrId.ByValue getFfiNameOrId(GraphNameOrId nameOrId) {
switch (nameOrId.getOpt()) {
case NAME:
return ArgUtils.asNameOrId(nameOrId.getName());
case ID:
default:
return ArgUtils.asNameOrId(nameOrId.getId());
}
}

private FfiConst.ByValue getFfiConst(RexNode rexNode) {
if (rexNode instanceof RexLiteral) {
return Utils.ffiConst((RexLiteral) rexNode);
}
throw new IllegalArgumentException("cannot convert rexNode=" + rexNode + " to ffi const");
private FfiPbPointer.ByValue ffiIndexPredicates(RexNode uniqueKeyFilters) {
GraphAlgebra.IndexPredicate indexPredicate =
uniqueKeyFilters.accept(
new RexToIndexPbConverter(true, this.isColumnId, this.rexBuilder));
return new FfiPbPointer.ByValue(indexPredicate.toByteArray());
}

private List<Integer> range(RexNode offset, RexNode fetch) {
Expand Down
Loading

0 comments on commit c19808d

Please sign in to comment.