Skip to content

Commit

Permalink
Merge pull request #150 from cirnoooo123/main
Browse files Browse the repository at this point in the history 
add secure KNN query framework
  • Loading branch information
@SongY123
SongY123 authored Apr 13, 2023
2 parents 3ba7773 + 63b39b1 commit 6310b0d
Show file tree
Hide file tree
Showing 6 changed files with 376 additions and 11 deletions.
56 changes: 49 additions & 7 deletions benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialBenchmarkTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public void testSqlSpatialDistance() throws SQLException {
}

@Test
public void testSqlSpatialDWithin() throws SQLException {
public void testSqlRangeQuery() throws SQLException {
String sql = "select * from spatial where DWithin(POINT(1404050, -4762163), S_POINT, 5)";
try (Statement stmt = user.createStatement()) {
ResultSet dataset = stmt.executeQuery(sql);
Expand All @@ -108,20 +108,62 @@ public void testSqlSpatialDWithin() throws SQLException {
}

@Test
public void testSqlSpatialKNN() throws SQLException {
String sql = "select * from spatial order by Distance(POINT(1404050, -4762163), S_POINT) asc limit 5";
public void testSqlRangeCount() throws SQLException {
String sql = "select count(*) from spatial where DWithin(POINT(1404050, -4762163), S_POINT, 5)";
try (Statement stmt = user.createStatement()) {
ResultSet dataset = stmt.executeQuery(sql);
long count = 0;
while (dataset.next()) {
printLine(dataset);
++count;
}
assertEquals(1, count);
dataset.close();
}
}

@Test
public void testSqlRangeJoin() throws SQLException {
String sql = "select * from spatial s1 join spatial s2 on DWithin(s1.S_POINT, s2.S_POINT, 5)";
try (Statement stmt = user.createStatement()) {
ResultSet dataset = stmt.executeQuery(sql);
long count = 0;
while (dataset.next()) {
printLine(dataset);
++count;
}
assertEquals(1, count);
dataset.close();
}
}

@Test
public void testSqlKNNQuery() throws SQLException {
String sql = "select * from spatial order by Distance(S_POINT, POINT(1404050, -4762163)) asc limit 10";
try (Statement stmt = user.createStatement()) {
ResultSet dataset = stmt.executeQuery(sql);
long count = 0;
while (dataset.next()) {
printLine(dataset);
++count;
}
assertEquals(10, count);
dataset.close();
}
}

@Test
public void testSqlKNNJOIN() throws SQLException {
String sql = "select * from spatial s1 join spatial s2 on KNN(s1.S_POINT, s2.S_POINT, 5)";
try (Statement stmt = user.createStatement()) {
ResultSet dataset = stmt.executeQuery(sql);
long count = 0;
while (dataset.next()) {
printLine(dataset);
++count;
}
assertEquals(1, count);
dataset.close();
/* TODO: 2023/4/2 error occurs when S_POINT is protected, expected 5, actual 15,
* Github Issue 115
*/
assertEquals(15, count);
}
}

Expand Down
107 changes: 103 additions & 4 deletions core/src/main/java/com/hufudb/openhufu/core/implementor/UserSideImplementor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,11 @@
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;

import com.hufudb.openhufu.core.implementor.spatial.knn.BinarySearchKNN;
import com.hufudb.openhufu.core.sql.plan.PlanUtils;
import com.hufudb.openhufu.data.schema.Schema;
import com.hufudb.openhufu.data.storage.DataSet;
import com.hufudb.openhufu.data.storage.LimitDataSet;
import com.hufudb.openhufu.data.storage.MultiSourceDataSet;
import com.hufudb.openhufu.data.storage.SortedDataSet;
import com.hufudb.openhufu.data.storage.*;
import com.hufudb.openhufu.data.storage.MultiSourceDataSet.Producer;
import com.hufudb.openhufu.implementor.PlanImplementor;
import com.hufudb.openhufu.interpreter.Interpreter;
Expand All @@ -22,6 +21,7 @@
import com.hufudb.openhufu.plan.UnaryPlan;
import com.hufudb.openhufu.proto.OpenHuFuData.DataSetProto;
import com.hufudb.openhufu.proto.OpenHuFuData.Modifier;
import com.hufudb.openhufu.proto.OpenHuFuPlan;
import com.hufudb.openhufu.proto.OpenHuFuPlan.PlanType;
import com.hufudb.openhufu.proto.OpenHuFuPlan.QueryPlanProto;
import org.apache.commons.lang3.tuple.Pair;
Expand Down Expand Up @@ -105,6 +105,9 @@ DataSet ownerSideQuery(Plan plan) {
@Override
public DataSet implement(Plan plan) {
if (isMultiParty(plan)) {
if (plan instanceof UnaryPlan && isMultiPartySecureKNN((UnaryPlan) plan)) {
return privacyKNN((UnaryPlan) plan);
}
// implement on owner side
DataSet dataset = ownerSideQuery(plan);
return dataset;
Expand All @@ -115,6 +118,102 @@ public DataSet implement(Plan plan) {
}
}

private DataSet privacyKNN(UnaryPlan plan) {
LOG.info("Using binary-search KNN.");
boolean USE_DP = false;
int k = plan.getFetch();
double left = 0;
double right = 1000000;
// if (USE_DP) {
right = kNNRadiusQuery(plan) * 2;
// }
double deviation = 1e-6;
int loop = 0;
long count = 0L;
if (USE_DP) {
while (left + deviation <= right) {
double mid = (left + right) / 2;
LOG.debug("k: {} left: {} right: {} mid: {}", k, left, right, mid);
Pair<Double, Double> res = dPRangeCount(plan);
count = Math.round(res.getKey());
if (Math.abs(res.getKey() - k) < res.getValue()) {
LOG.debug("change method on loop {}", loop);
break;
}
if (count > k) {
right = mid;
} else if (count < k) {
left = mid;
}
loop++;
LOG.debug("loop {} with result size {}", loop, count);
}
}
while (left + deviation <= right) {
double mid = (left + right) / 2;
// int sign = privacyCompare(plan);
int sign = (int) privacyCompare(plan, mid, k);
LOG.debug("loop {} with sign {}", loop, sign);
if (sign < 0) {
left = mid;
} else if (sign > 0) {
right = mid;
} else {
loop++;
return kNNCircleRangeQuery(plan, mid);
}
loop++;
}
return kNNCircleRangeQuery(plan, right);
}
private double kNNRadiusQuery(UnaryPlan plan) {
//todo -sjz
DataSetIterator dataSet = ownerSideQuery(BinarySearchKNN.generateKNNRadiusQueryPlan(plan)).getIterator();
double right = 1000000;
while (dataSet.next()) {
double res = (double) dataSet.get(0);
LOG.info(String.valueOf(res));
if (right > res) {
right = res;
}
}
return right;
}
private Pair<Double, Double> dPRangeCount(UnaryPlan plan) {
//todo -sjz
ownerSideQuery(BinarySearchKNN.generateDPRangeCountPlan(plan));
return null;
}
private long privacyCompare(UnaryPlan plan, double range, long k) {
//todo -sjz now it is using secretSharingSum
DataSetIterator dataSet = ownerSideQuery(BinarySearchKNN.generatePrivacyComparePlan(plan, range)).getIterator();
dataSet.next();
long res = (long) dataSet.get(0);
return res - k;
}
private DataSet kNNCircleRangeQuery(UnaryPlan plan, double range) {
return ownerSideQuery(BinarySearchKNN.generateKNNCircleRangeQueryPlan(plan, range));
}
private boolean isMultiPartySecureKNN(UnaryPlan unary) {
LeafPlan leaf = (LeafPlan) unary.getChildren().get(0);
boolean hasLimit = leaf.getOffset() != 0 || leaf.getFetch() != 0;
if (!hasLimit) {
return false;
}
if (leaf.getOrders() == null || leaf.getOrders().size() < 1) {
return false;
}
int orderRef = leaf.getOrders().get(0).getRef();
if (!(leaf.getSelectExps().get(orderRef).getOpType().equals(OpenHuFuPlan.OperatorType.SCALAR_FUNC)
&& leaf.getSelectExps().get(orderRef).getStr().equals("distance"))) {
return false;
}
if (leaf.getOrders().get(0).getDirection().equals(OpenHuFuPlan.Direction.ASC)) {
LOG.info("This is a KNN query.");
return true;
}
return false;
}
@Override
public DataSet binaryQuery(BinaryPlan binary) {
List<Plan> children = binary.getChildren();
Expand Down
85 changes: 85 additions & 0 deletions core/src/main/java/com/hufudb/openhufu/core/implementor/spatial/knn/BinarySearchKNN.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package com.hufudb.openhufu.core.implementor.spatial.knn;

import com.google.common.collect.ImmutableList;
import com.hufudb.openhufu.expression.AggFuncType;
import com.hufudb.openhufu.expression.ExpressionFactory;
import com.hufudb.openhufu.plan.LeafPlan;
import com.hufudb.openhufu.plan.Plan;
import com.hufudb.openhufu.plan.UnaryPlan;
import com.hufudb.openhufu.proto.OpenHuFuData;
import com.hufudb.openhufu.proto.OpenHuFuPlan;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;

public class BinarySearchKNN {
static final Logger LOG = LoggerFactory.getLogger(BinarySearchKNN.class);

public static Plan generateKNNRadiusQueryPlan(UnaryPlan originalPlan) {
LOG.info("rewriting KNNRadiusQueryPlan");
LeafPlan originalLeaf = (LeafPlan) originalPlan.getChildren().get(0);
LeafPlan leafPlan = new LeafPlan();
leafPlan.setTableName(originalLeaf.getTableName());
OpenHuFuPlan.Expression distance = originalLeaf.getSelectExps()
.get(originalLeaf.getOrders().get(0).getRef());
leafPlan.setSelectExps(ImmutableList.of(distance));
leafPlan.setOrders(ImmutableList.of(OpenHuFuPlan.Collation.newBuilder().setRef(0)
.setDirection(OpenHuFuPlan.Direction.ASC).build()));
leafPlan.setOffset(originalLeaf.getFetch() - 1);
leafPlan.setFetch(1);
LOG.info(leafPlan.toString());
return leafPlan;
}
public static Plan generateDPRangeCountPlan(UnaryPlan originalPlan) {
return null;
}
public static Plan generatePrivacyComparePlan(UnaryPlan originalPlan, double range) {
LOG.info("rewriting PrivacyComparePlan");
LeafPlan originalLeaf = (LeafPlan) originalPlan.getChildren().get(0);
LeafPlan leafPlan = new LeafPlan();
leafPlan.setTableName(originalLeaf.getTableName());
leafPlan.setSelectExps(originalLeaf.getSelectExps());
OpenHuFuPlan.Expression distance = originalLeaf.getSelectExps()
.get(originalLeaf.getOrders().get(0).getRef());
OpenHuFuPlan.Expression dwithin = ExpressionFactory.createScalarFunc(OpenHuFuData.ColumnType.BOOLEAN, "dwithin",
ImmutableList.of(distance.getIn(0),
distance.getIn(1),
ExpressionFactory.createLiteral(OpenHuFuData.ColumnType.DOUBLE, range)));


List<OpenHuFuPlan.Expression> whereExps = ImmutableList.of(dwithin);
leafPlan.setWhereExps(whereExps);
leafPlan.setAggExps(ImmutableList.of(ExpressionFactory.createAggFunc(OpenHuFuData.ColumnType.LONG,
OpenHuFuData.Modifier.PROTECTED, AggFuncType.COUNT.getId(), ImmutableList.of())));
leafPlan.setOrders(originalLeaf.getOrders());

UnaryPlan unaryPlan = new UnaryPlan(leafPlan);
unaryPlan.setSelectExps(ImmutableList.of(ExpressionFactory
.createInputRef(0, OpenHuFuData.ColumnType.LONG, OpenHuFuData.Modifier.PROTECTED)));
unaryPlan.setAggExps(ImmutableList.of(ExpressionFactory
.createAggFunc(OpenHuFuData.ColumnType.LONG, OpenHuFuData.Modifier.PROTECTED, AggFuncType.SUM.getId(),
ImmutableList.of(ExpressionFactory
.createInputRef(0, OpenHuFuData.ColumnType.LONG, OpenHuFuData.Modifier.PROTECTED)))));
LOG.info(unaryPlan.toString());
return unaryPlan;
}
public static Plan generateKNNCircleRangeQueryPlan(UnaryPlan originalPlan, double range) {
LeafPlan originalLeaf = (LeafPlan) originalPlan.getChildren().get(0);
LeafPlan leafPlan = new LeafPlan();
leafPlan.setTableName(originalLeaf.getTableName());
leafPlan.setSelectExps(originalLeaf.getSelectExps());
OpenHuFuPlan.Expression distance = originalLeaf.getSelectExps()
.get(originalLeaf.getOrders().get(0).getRef());
OpenHuFuPlan.Expression dwithin = ExpressionFactory.createScalarFunc(OpenHuFuData.ColumnType.BOOLEAN, "dwithin",
ImmutableList.of(distance.getIn(0),
distance.getIn(1),
ExpressionFactory.createLiteral(OpenHuFuData.ColumnType.DOUBLE, range)));
LOG.info("rewriting KNNCircleRangeQueryPlan");
List<OpenHuFuPlan.Expression> whereExps = ImmutableList.of(dwithin);
leafPlan.setWhereExps(whereExps);
leafPlan.setOrders(originalLeaf.getOrders());
LOG.info(leafPlan.toString());
return leafPlan;
}
}
Loading

0 comments on commit 6310b0d

Please sign in to comment.