Skip to content

Commit

Permalink
Improve set inclusion kernel performance (deephaven#5300)
Browse files Browse the repository at this point in the history 
* Improved set kernel performance

* Address PR comments.
  • Loading branch information
@lbooker42
lbooker42 authored Mar 27, 2024
1 parent feb6024 commit 27cd680
Show file tree
Hide file tree
Showing 10 changed files with 348 additions and 142 deletions.
12 changes: 3 additions & 9 deletions engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -537,21 +537,15 @@ private WritableRowSet filterLinear(final RowSet selection, final boolean filter
// @formatter:off
try (final ColumnSource.GetContext keyGetContext = sourceKeySource.makeGetContext(maxChunkSize);
final RowSequence.Iterator selectionIterator = selection.getRowSequenceIterator();
final WritableBooleanChunk<Values> matches = WritableBooleanChunk.makeWritableChunk(maxChunkSize)) {
final WritableLongChunk<OrderedRowKeys> matchingKeys = WritableLongChunk.makeWritableChunk(maxChunkSize)) {
// @formatter:on

while (selectionIterator.hasMore()) {
final RowSequence selectionChunk = selectionIterator.getNextRowSequenceWithLength(maxChunkSize);
final LongChunk<OrderedRowKeys> selectionRowKeyChunk = selectionChunk.asRowKeyChunk();
final Chunk<Values> keyChunk = Chunk.downcast(sourceKeySource.getChunk(keyGetContext, selectionChunk));
setKernel.matchValues(keyChunk, matches, filterInclusion);

final int thisChunkSize = keyChunk.size();
for (int ii = 0; ii < thisChunkSize; ++ii) {
if (matches.get(ii)) {
filteredRowSetBuilder.appendKey(selectionRowKeyChunk.get(ii));
}
}
setKernel.matchValues(keyChunk, selectionRowKeyChunk, matchingKeys, filterInclusion);
filteredRowSetBuilder.appendOrderedRowKeysChunk(matchingKeys);
}
}

Expand Down
58 changes: 42 additions & 16 deletions .../main/java/io/deephaven/engine/table/impl/select/setinclusion/ByteSetInclusionKernel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,9 @@
package io.deephaven.engine.table.impl.select.setinclusion;

import gnu.trove.iterator.TByteIterator;
import io.deephaven.chunk.ByteChunk;
import io.deephaven.chunk.Chunk;
import io.deephaven.chunk.WritableBooleanChunk;
import io.deephaven.chunk.*;
import io.deephaven.chunk.attributes.Values;
import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys;
import io.deephaven.util.type.TypeUtils;
import gnu.trove.set.TByteSet;
import gnu.trove.set.hash.TByteHashSet;
Expand All @@ -24,24 +23,24 @@ public class ByteSetInclusionKernel implements SetInclusionKernel {
private final TByteSet liveValues;
private final boolean inclusion;

ByteSetInclusionKernel(Collection<Object> liveValues, boolean inclusion) {
ByteSetInclusionKernel(@NotNull final Collection<Object> liveValues, final boolean inclusion) {
this.liveValues = new TByteHashSet(liveValues.size());
liveValues.forEach(x -> this.liveValues.add(TypeUtils.unbox((Byte) x)));
this.inclusion = inclusion;
}

ByteSetInclusionKernel(boolean inclusion) {
ByteSetInclusionKernel(final boolean inclusion) {
this.liveValues = new TByteHashSet();
this.inclusion = inclusion;
}

@Override
public boolean add(Object key) {
public boolean add(@NotNull final Object key) {
return liveValues.add(TypeUtils.unbox((Byte) key));
}

@Override
public boolean remove(Object key) {
public boolean remove(@NotNull final Object key) {
return liveValues.remove(TypeUtils.unbox((Byte) key));
}

Expand Down Expand Up @@ -75,22 +74,49 @@ public java.util.Iterator<Object> iterator() {
}

@Override
public void matchValues(Chunk<Values> values, WritableBooleanChunk<?> matches) {
matchValues(values.asByteChunk(), matches, inclusion);
public void matchValues(
@NotNull final Chunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results) {
matchValues(values, keys, results, inclusion);
}

@Override
public void matchValues(Chunk<Values> values, WritableBooleanChunk<?> matches, boolean inclusionOverride) {
matchValues(values.asByteChunk(), matches, inclusionOverride);
public void matchValues(
@NotNull final Chunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results,
final boolean inclusionOverride) {
if (inclusionOverride) {
matchValues(values.asByteChunk(), keys, results);
} else {
matchValuesInvert(values.asByteChunk(), keys, results);
}
}

private void matchValues(
ByteChunk<Values> values,
WritableBooleanChunk<?> matches,
boolean inclusionToUse) {
@NotNull final ByteChunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results) {
results.setSize(0);
for (int ii = 0; ii < values.size(); ++ii) {
final byte checkValue = values.get(ii);
if (liveValues.contains(checkValue)) {
results.add(keys.get(ii));
}
}
}

private void matchValuesInvert(
@NotNull final ByteChunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results) {
results.setSize(0);
for (int ii = 0; ii < values.size(); ++ii) {
matches.set(ii, liveValues.contains(values.get(ii)) == inclusionToUse);
final byte checkValue = values.get(ii);
if (!liveValues.contains(checkValue)) {
results.add(keys.get(ii));
}
}
matches.setSize(values.size());
}
}
58 changes: 42 additions & 16 deletions .../main/java/io/deephaven/engine/table/impl/select/setinclusion/CharSetInclusionKernel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,9 @@
package io.deephaven.engine.table.impl.select.setinclusion;

import gnu.trove.iterator.TCharIterator;
import io.deephaven.chunk.CharChunk;
import io.deephaven.chunk.Chunk;
import io.deephaven.chunk.WritableBooleanChunk;
import io.deephaven.chunk.*;
import io.deephaven.chunk.attributes.Values;
import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys;
import io.deephaven.util.type.TypeUtils;
import gnu.trove.set.TCharSet;
import gnu.trove.set.hash.TCharHashSet;
Expand All @@ -20,24 +19,24 @@ public class CharSetInclusionKernel implements SetInclusionKernel {
private final TCharSet liveValues;
private final boolean inclusion;

CharSetInclusionKernel(Collection<Object> liveValues, boolean inclusion) {
CharSetInclusionKernel(@NotNull final Collection<Object> liveValues, final boolean inclusion) {
this.liveValues = new TCharHashSet(liveValues.size());
liveValues.forEach(x -> this.liveValues.add(TypeUtils.unbox((Character) x)));
this.inclusion = inclusion;
}

CharSetInclusionKernel(boolean inclusion) {
CharSetInclusionKernel(final boolean inclusion) {
this.liveValues = new TCharHashSet();
this.inclusion = inclusion;
}

@Override
public boolean add(Object key) {
public boolean add(@NotNull final Object key) {
return liveValues.add(TypeUtils.unbox((Character) key));
}

@Override
public boolean remove(Object key) {
public boolean remove(@NotNull final Object key) {
return liveValues.remove(TypeUtils.unbox((Character) key));
}

Expand Down Expand Up @@ -71,22 +70,49 @@ public java.util.Iterator<Object> iterator() {
}

@Override
public void matchValues(Chunk<Values> values, WritableBooleanChunk<?> matches) {
matchValues(values.asCharChunk(), matches, inclusion);
public void matchValues(
@NotNull final Chunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results) {
matchValues(values, keys, results, inclusion);
}

@Override
public void matchValues(Chunk<Values> values, WritableBooleanChunk<?> matches, boolean inclusionOverride) {
matchValues(values.asCharChunk(), matches, inclusionOverride);
public void matchValues(
@NotNull final Chunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results,
final boolean inclusionOverride) {
if (inclusionOverride) {
matchValues(values.asCharChunk(), keys, results);
} else {
matchValuesInvert(values.asCharChunk(), keys, results);
}
}

private void matchValues(
CharChunk<Values> values,
WritableBooleanChunk<?> matches,
boolean inclusionToUse) {
@NotNull final CharChunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results) {
results.setSize(0);
for (int ii = 0; ii < values.size(); ++ii) {
final char checkValue = values.get(ii);
if (liveValues.contains(checkValue)) {
results.add(keys.get(ii));
}
}
}

private void matchValuesInvert(
@NotNull final CharChunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results) {
results.setSize(0);
for (int ii = 0; ii < values.size(); ++ii) {
matches.set(ii, liveValues.contains(values.get(ii)) == inclusionToUse);
final char checkValue = values.get(ii);
if (!liveValues.contains(checkValue)) {
results.add(keys.get(ii));
}
}
matches.setSize(values.size());
}
}
58 changes: 42 additions & 16 deletions ...ain/java/io/deephaven/engine/table/impl/select/setinclusion/DoubleSetInclusionKernel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,9 @@
package io.deephaven.engine.table.impl.select.setinclusion;

import gnu.trove.iterator.TDoubleIterator;
import io.deephaven.chunk.DoubleChunk;
import io.deephaven.chunk.Chunk;
import io.deephaven.chunk.WritableBooleanChunk;
import io.deephaven.chunk.*;
import io.deephaven.chunk.attributes.Values;
import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys;
import io.deephaven.util.type.TypeUtils;
import gnu.trove.set.TDoubleSet;
import gnu.trove.set.hash.TDoubleHashSet;
Expand All @@ -24,24 +23,24 @@ public class DoubleSetInclusionKernel implements SetInclusionKernel {
private final TDoubleSet liveValues;
private final boolean inclusion;

DoubleSetInclusionKernel(Collection<Object> liveValues, boolean inclusion) {
DoubleSetInclusionKernel(@NotNull final Collection<Object> liveValues, final boolean inclusion) {
this.liveValues = new TDoubleHashSet(liveValues.size());
liveValues.forEach(x -> this.liveValues.add(TypeUtils.unbox((Double) x)));
this.inclusion = inclusion;
}

DoubleSetInclusionKernel(boolean inclusion) {
DoubleSetInclusionKernel(final boolean inclusion) {
this.liveValues = new TDoubleHashSet();
this.inclusion = inclusion;
}

@Override
public boolean add(Object key) {
public boolean add(@NotNull final Object key) {
return liveValues.add(TypeUtils.unbox((Double) key));
}

@Override
public boolean remove(Object key) {
public boolean remove(@NotNull final Object key) {
return liveValues.remove(TypeUtils.unbox((Double) key));
}

Expand Down Expand Up @@ -75,22 +74,49 @@ public java.util.Iterator<Object> iterator() {
}

@Override
public void matchValues(Chunk<Values> values, WritableBooleanChunk<?> matches) {
matchValues(values.asDoubleChunk(), matches, inclusion);
public void matchValues(
@NotNull final Chunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results) {
matchValues(values, keys, results, inclusion);
}

@Override
public void matchValues(Chunk<Values> values, WritableBooleanChunk<?> matches, boolean inclusionOverride) {
matchValues(values.asDoubleChunk(), matches, inclusionOverride);
public void matchValues(
@NotNull final Chunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results,
final boolean inclusionOverride) {
if (inclusionOverride) {
matchValues(values.asDoubleChunk(), keys, results);
} else {
matchValuesInvert(values.asDoubleChunk(), keys, results);
}
}

private void matchValues(
DoubleChunk<Values> values,
WritableBooleanChunk<?> matches,
boolean inclusionToUse) {
@NotNull final DoubleChunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results) {
results.setSize(0);
for (int ii = 0; ii < values.size(); ++ii) {
final double checkValue = values.get(ii);
if (liveValues.contains(checkValue)) {
results.add(keys.get(ii));
}
}
}

private void matchValuesInvert(
@NotNull final DoubleChunk<Values> values,
@NotNull final LongChunk<OrderedRowKeys> keys,
@NotNull WritableLongChunk<OrderedRowKeys> results) {
results.setSize(0);
for (int ii = 0; ii < values.size(); ++ii) {
matches.set(ii, liveValues.contains(values.get(ii)) == inclusionToUse);
final double checkValue = values.get(ii);
if (!liveValues.contains(checkValue)) {
results.add(keys.get(ii));
}
}
matches.setSize(values.size());
}
}
Loading

0 comments on commit 27cd680

Please sign in to comment.