diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java index 43e20c8d9b1..91dbc7e323b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java @@ -113,6 +113,21 @@ public SafeCloseable beginOperation(@NotNull final QueryTable parent) { void initializeFilters(@NotNull QueryTable parent) { final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); Arrays.stream(whereFilters).forEach(filter -> filter.init(parent.getDefinition(), compilationProcessor)); + + final List disallowedRowVariables = + Arrays.stream(whereFilters).filter(WhereFilter::hasVirtualRowVariables).collect(Collectors.toList()); + if (!disallowedRowVariables.isEmpty()) { + throw new UncheckedTableException( + "wouldMatch filters cannot use virtual row variables (i, ii, and k): " + disallowedRowVariables); + } + + final List disallowedColumnVectors = + Arrays.stream(whereFilters).filter(wf -> !wf.getColumnArrays().isEmpty()).collect(Collectors.toList()); + if (!disallowedColumnVectors.isEmpty()) { + throw new UncheckedTableException( + "wouldMatch filters cannot use column Vectors (_ syntax): " + disallowedColumnVectors); + } + compilationProcessor.compile(); } @@ -517,13 +532,14 @@ private RowSet update(RowSet added, RowSet removed, RowSet modified, try (final SafeCloseableList toClose = new SafeCloseableList()) { // Filter and add addeds - final WritableRowSet filteredAdded = toClose.add(filter.filter(added, source, table, false)); + final WritableRowSet filteredAdded = toClose.add(filter.filter(added, table.getRowSet(), table, false)); RowSet keysToRemove = EMPTY_INDEX; // If we were affected, recompute mods and re-add the ones that pass. if (affected) { downstreamModified.setAll(name); - final RowSet filteredModified = toClose.add(filter.filter(modified, source, table, false)); + final RowSet filteredModified = + toClose.add(filter.filter(modified, table.getRowSet(), table, false)); // Now apply the additions and remove any non-matching modifieds filteredAdded.insert(filteredModified); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java index 2007eee6526..9b9602843f5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java @@ -311,6 +311,11 @@ public boolean hasConstantArrayAccess() { return getFormulaShiftColPair() != null; } + @Override + public boolean hasVirtualRowVariables() { + return usesI || usesII || usesK; + } + /** * @return a Pair object, consisting of formula string and shift to column MatchPairs, if the filter formula or * expression has Array Access that conforms to "i +/- <constant>" or "ii +/- <constant>". If diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FilterSelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FilterSelectColumn.java new file mode 100644 index 00000000000..2b096a69002 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FilterSelectColumn.java @@ -0,0 +1,310 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.table.impl.select; + +import io.deephaven.api.filter.Filter; +import io.deephaven.chunk.ChunkType; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableObjectChunk; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.exceptions.UncheckedTableException; +import io.deephaven.engine.rowset.*; +import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.engine.table.impl.QueryTable; +import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; +import io.deephaven.engine.table.impl.sources.SparseArrayColumnSource; +import io.deephaven.engine.table.impl.sources.ViewColumnSource; +import org.jetbrains.annotations.NotNull; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * The FilterSelectColumn wraps a {@link Filter} and producing a column of true or false Boolean values described by the + * filter. + *

+ * This SelectColumn is appropriate as an argument to a {@link Table#view(Collection)} or + * {@link Table#updateView(String...)}, lazily evaluating the equivalent of a {@link Table#wouldMatch(String...)} + * operation. Although select and update can also use Filters, wouldMatch provides a more efficient path for realized + * results as it stores and updates only a {@link RowSet}. The FilterSelectColumn can only evaluate the Filter one chunk + * at a time, and must write to an in-memory {@link Boolean} {@link ColumnSource}. + */ +class FilterSelectColumn implements SelectColumn { + + // We don't actually care to do anything, but cannot return null + private static final Formula.FillContext FILL_CONTEXT_INSTANCE = new Formula.FillContext() {}; + + @NotNull + private final String destName; + @NotNull + private final WhereFilter filter; + + private Table tableToFilter; + /** + * We store a copy of our table definition, to ensure that it is identical between initDef and initInputs. + */ + private TableDefinition computedDefinition; + + /** + * Create a FilterSelectColumn with the given name and {@link WhereFilter}. + * + * @param destName the name of the result column + * @param filter the filter that is evaluated to true or false for each row of the table + * @return a new FilterSelectColumn representing the provided filter. + */ + static FilterSelectColumn of(@NotNull final String destName, @NotNull final Filter filter) { + return new FilterSelectColumn(destName, WhereFilter.of(filter)); + } + + private FilterSelectColumn(@NotNull final String destName, @NotNull final WhereFilter filter) { + this.destName = destName; + this.filter = filter; + } + + @Override + public String toString() { + return "filter(" + filter + ')'; + } + + @Override + public List initInputs(final TrackingRowSet rowSet, + final Map> columnsOfInterest) { + tableToFilter = new QueryTable(rowSet, columnsOfInterest); + if (!computedDefinition.equals(tableToFilter.getDefinition())) { + throw new IllegalStateException( + "Definition changed between initDef and initInputs in FilterSelectColumn: initDef=" + + computedDefinition + ", initInputs" + tableToFilter.getDefinition()); + } + return filter.getColumns(); + } + + @Override + public List initDef(@NotNull final Map> columnDefinitionMap) { + filter.init(computedDefinition = TableDefinition.of(columnDefinitionMap.values())); + return checkForInvalidFilters(); + } + + @Override + public List initDef(@NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + filter.init(computedDefinition = TableDefinition.of(columnDefinitionMap.values()), compilationRequestProcessor); + return checkForInvalidFilters(); + } + + /** + * Validates the filter to ensure it does not contain invalid filters such as column vectors or virtual row + * variables. Throws an {@link UncheckedTableException} if any invalid filters are found. + * + * @return the list of columns required by the filter. + */ + private List checkForInvalidFilters() { + if (!filter.getColumnArrays().isEmpty()) { + throw new UncheckedTableException( + "Cannot use a filter with column Vectors (_ syntax) in select, view, update, or updateView: " + + filter); + } + if (filter.hasVirtualRowVariables()) { + throw new UncheckedTableException( + "Cannot use a filter with virtual row variables (i, ii, or k) in select, view, update, or updateView: " + + filter); + } + if (filter.isRefreshing()) { + /* + * TODO: DH-18052: updateView and view should support refreshing Filter Expressions + * + * This would enable us to use a whereIn or whereNotIn for things like conditional formatting; which could + * be attractive. However, a join or wouldMatch gets you there without the additional complexity. + * + * Supporting this requires SelectColumn dependencies, which have not previously existed. Additionally, if + * we were to support these for select and update (as opposed to view and updateView), then the filter could + * require recomputing the entire result table whenever anything changes. + */ + throw new UncheckedTableException( + "Cannot use a refreshing filter in select, view, update, or updateView: " + filter); + } + + return filter.getColumns(); + } + + @Override + public Class getReturnedType() { + return Boolean.class; + } + + @Override + public Class getReturnedComponentType() { + return null; + } + + @Override + public List getColumns() { + return filter.getColumns(); + } + + @Override + public List getColumnArrays() { + /* This should always be empty, because initDef throws when arrays or virtual row variables are used. */ + return List.of(); + } + + @Override + public boolean hasVirtualRowVariables() { + /* This should always be false, because initDef throws when arrays or ii and friends are used. */ + return false; + } + + @NotNull + @Override + public ColumnSource getDataView() { + return new ViewColumnSource<>(Boolean.class, new FilterFormula(), isStateless()); + } + + @NotNull + @Override + public ColumnSource getLazyView() { + return getDataView(); + } + + @Override + public String getName() { + return destName; + } + + @Override + public MatchPair getMatchPair() { + throw new UnsupportedOperationException(); + } + + @Override + public final WritableColumnSource newDestInstance(final long size) { + return SparseArrayColumnSource.getSparseMemoryColumnSource(size, Boolean.class); + } + + @Override + public final WritableColumnSource newFlatDestInstance(final long size) { + return InMemoryColumnSource.getImmutableMemoryColumnSource(size, Boolean.class, null); + } + + @Override + public boolean isRetain() { + return false; + } + + @Override + public boolean isStateless() { + return filter.permitParallelization(); + } + + @Override + public FilterSelectColumn copy() { + return new FilterSelectColumn(destName, filter.copy()); + } + + @Override + public void validateSafeForRefresh(final BaseTable sourceTable) { + filter.validateSafeForRefresh(sourceTable); + } + + private class FilterFormula extends Formula { + public FilterFormula() { + super(null); + } + + @Override + public Boolean getBoolean(final long rowKey) { + try (final WritableRowSet selection = RowSetFactory.fromKeys(rowKey); + final WritableRowSet filteredRowSet = + filter.filter(selection, tableToFilter.getRowSet(), tableToFilter, false)) { + return filteredRowSet.isNonempty(); + } + } + + @Override + public Boolean getPrevBoolean(final long rowKey) { + try (final WritableRowSet selection = RowSetFactory.fromKeys(rowKey); + final WritableRowSet filteredRowSet = filter.filter(selection, + tableToFilter.getRowSet().prev(), tableToFilter, true)) { + return filteredRowSet.isNonempty(); + } + } + + @Override + public Object get(final long rowKey) { + return getBoolean(rowKey); + } + + @Override + public Object getPrev(final long rowKey) { + return getPrevBoolean(rowKey); + } + + @Override + public ChunkType getChunkType() { + return ChunkType.Object; + } + + @Override + public FillContext makeFillContext(final int chunkCapacity) { + return FILL_CONTEXT_INSTANCE; + } + + @Override + public void fillChunk( + @NotNull final FillContext fillContext, + @NotNull final WritableChunk destination, + @NotNull final RowSequence rowSequence) { + doFill(rowSequence, destination, false); + } + + @Override + public void fillPrevChunk( + @NotNull final FillContext fillContext, + @NotNull final WritableChunk destination, + @NotNull final RowSequence rowSequence) { + doFill(rowSequence, destination, true); + } + + private void doFill(@NotNull final RowSequence rowSequence, final WritableChunk destination, + final boolean usePrev) { + final WritableObjectChunk booleanDestination = destination.asWritableObjectChunk(); + booleanDestination.setSize(rowSequence.intSize()); + final RowSet fullSet = usePrev ? tableToFilter.getRowSet().prev() : tableToFilter.getRowSet(); + + try (final RowSet inputRowSet = rowSequence.asRowSet(); + final RowSet filtered = filter.filter(inputRowSet, fullSet, tableToFilter, usePrev)) { + if (filtered.size() == inputRowSet.size()) { + // if everything matches, short circuit the iteration + booleanDestination.fillWithValue(0, booleanDestination.size(), true); + return; + } + + int offset = 0; + + try (final RowSequence.Iterator inputRows = inputRowSet.getRowSequenceIterator(); + final RowSet.Iterator trueRows = filtered.iterator()) { + while (trueRows.hasNext()) { + final long nextTrue = trueRows.nextLong(); + // Find all the false rows between the last consumed input row and the next true row + final int falsesSkipped = (int) inputRows.advanceAndGetPositionDistance(nextTrue + 1) - 1; + if (falsesSkipped > 0) { + booleanDestination.fillWithValue(offset, falsesSkipped, false); + offset += falsesSkipped; + } + booleanDestination.set(offset++, true); + } + } + + final int remainingFalses = booleanDestination.size() - offset; + // Fill everything else up with false, because we've exhausted the trues + if (remainingFalses > 0) { + booleanDestination.fillWithValue(offset, remainingFalses, false); + } + } + } + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java index 249ec9a14f2..63239a0a461 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java @@ -278,7 +278,7 @@ public SelectColumn visit(Literal rhs) { @Override public SelectColumn visit(Filter rhs) { - return makeSelectColumn(Strings.of(rhs)); + return FilterSelectColumn.of(lhs.name(), rhs); } @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java index 9dae903ae4e..454cc547020 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java @@ -301,6 +301,13 @@ default boolean canMemoize() { return false; } + /** + * Returns true if this filter uses row virtual offset columns of {@code i}, {@code ii} or {@code k}. + */ + default boolean hasVirtualRowVariables() { + return false; + } + /** * Create a copy of this WhereFilter. * @@ -331,7 +338,7 @@ default Filter invert() { @Override default T walk(Expression.Visitor visitor) { - throw new UnsupportedOperationException("WhereFilters do not implement walk"); + return visitor.visit(this); } @Override diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableSelectUpdateTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableSelectUpdateTest.java index c8eb9073dbe..10e1b015cf5 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableSelectUpdateTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableSelectUpdateTest.java @@ -3,22 +3,23 @@ // package io.deephaven.engine.table.impl; -import io.deephaven.api.JoinMatch; -import io.deephaven.api.TableOperations; +import io.deephaven.api.*; +import io.deephaven.api.filter.Filter; +import io.deephaven.api.filter.FilterIn; +import io.deephaven.api.literal.Literal; import io.deephaven.base.testing.BaseArrayTestCase; +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.chunk.attributes.Values; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryScope; +import io.deephaven.engine.exceptions.UncheckedTableException; import io.deephaven.engine.liveness.LivenessScope; import io.deephaven.engine.liveness.LivenessScopeStack; +import io.deephaven.engine.primitive.iterator.CloseableIterator; import io.deephaven.engine.rowset.*; -import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.table.ShiftObliviousListener; -import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.impl.select.DhFormulaColumn; -import io.deephaven.engine.table.impl.select.FormulaCompilationException; -import io.deephaven.engine.table.impl.select.SelectColumn; -import io.deephaven.engine.table.impl.select.SelectColumnFactory; +import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.select.*; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.engine.table.impl.sources.LongSparseArraySource; import io.deephaven.engine.table.impl.sources.RedirectedColumnSource; @@ -42,9 +43,12 @@ import org.junit.Rule; import org.junit.Test; +import java.text.DecimalFormat; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static io.deephaven.engine.testutil.TstUtils.*; import static io.deephaven.engine.util.TableTools.*; @@ -1302,6 +1306,249 @@ public void testPropagationOfAttributes() { } } + @Test + public void testFilterExpression() { + final Filter filter = FilterIn.of(ColumnName.of("A"), Literal.of(1), Literal.of(3)); + final Table t = TableTools.newTable(intCol("A", 1, 1, 2, 3, 5, 8)); + final Table wm = t.wouldMatch(new WouldMatchPair("AWM", filter)); + + // use an update + final Table up = t.update(List.of(Selectable.of(ColumnName.of("AWM"), filter))); + assertTableEquals(wm, up); + + // use an updateView + final Table upv = t.updateView(List.of(Selectable.of(ColumnName.of("AWM"), filter))); + assertTableEquals(wm, upv); + + // Test the getBoolean method + assertEquals(true, upv.getColumnSource("AWM").get(t.getRowSet().get(0))); + assertEquals(false, upv.getColumnSource("AWM").get(t.getRowSet().get(2))); + + // and now a more generic WhereFilter + + final Filter filter2 = WhereFilterFactory.getExpression("A == 1 || A==3"); + final Table wm2 = t.wouldMatch(new WouldMatchPair("AWM", filter2)); + + // use an update + final Table up2 = t.update(List.of(Selectable.of(ColumnName.of("AWM"), filter2))); + assertTableEquals(wm2, up2); + + // a Filter where nothing is true, to check that state + final Table upvf = t.updateView(List.of(Selectable.of(ColumnName.of("AWM"), Filter.ofFalse()))); + assertTableEquals(t.updateView("AWM=false"), upvf); + + // a Filter where everything is true + final Table upvt = t.updateView(List.of(Selectable.of(ColumnName.of("AWM"), Filter.ofTrue()))); + assertTableEquals(t.updateView("AWM=true"), upvt); + + // a Filter where the last value in the chunk is true + final Filter filter3 = WhereFilterFactory.getExpression("A in 8"); + final Table wm3 = t.wouldMatch(new WouldMatchPair("AWM", filter3)); + final Table upv3 = t.updateView(List.of(Selectable.of(ColumnName.of("AWM"), filter3))); + assertTableEquals(wm3, upv3); + } + + @Test + public void testFilterExpressionGetPrev() { + final Filter filter = FilterIn.of(ColumnName.of("A"), Literal.of(2), Literal.of(4)); + final QueryTable t = TstUtils.testRefreshingTable(i(2, 4, 6, 8).toTracking(), intCol("A", 1, 2, 3, 4)); + // noinspection resource + final TrackingWritableRowSet rs = t.getRowSet().writableCast(); + + // use an updateView + final Table upv = t.updateView(List.of(Selectable.of(ColumnName.of("AWM"), filter))); + + // Test the getBoolean method + ColumnSource resultColumn = upv.getColumnSource("AWM"); + assertEquals(false, resultColumn.get(rs.get(0))); + assertEquals(true, resultColumn.get(rs.get(1))); + assertEquals(false, resultColumn.get(rs.get(2))); + assertEquals(true, resultColumn.get(rs.get(3))); + + // and do it with chunks + try (final CloseableIterator awm = upv.columnIterator("AWM")) { + assertEquals(Arrays.asList(false, true, false, true), awm.stream().collect(Collectors.toList())); + } + + final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast(); + + updateGraph.runWithinUnitTestCycle(() -> { + assertEquals(false, resultColumn.get(t.getRowSet().get(0))); + assertEquals(true, resultColumn.get(t.getRowSet().get(1))); + assertEquals(false, resultColumn.get(t.getRowSet().get(2))); + assertEquals(true, resultColumn.get(t.getRowSet().get(3))); + + final RowSet prevRowset = rs.prev(); + assertEquals(false, resultColumn.getPrev(prevRowset.get(0))); + assertEquals(true, resultColumn.getPrev(prevRowset.get(1))); + assertEquals(false, resultColumn.getPrev(prevRowset.get(2))); + assertEquals(true, resultColumn.getPrev(prevRowset.get(3))); + + addToTable(t, i(1, 2, 9), intCol("A", 2, 2, 4)); + removeRows(t, i(8)); + rs.insert(i(1, 9)); + rs.remove(8); + t.notifyListeners(i(1, 9), i(8), i()); + + // with a chunk + try (final ChunkSource.GetContext fc = resultColumn.makeGetContext(4)) { + final ObjectChunk prevValues = + resultColumn.getPrevChunk(fc, prevRowset).asObjectChunk(); + assertEquals(false, prevValues.get(0)); + assertEquals(true, prevValues.get(1)); + assertEquals(false, prevValues.get(2)); + assertEquals(true, prevValues.get(3)); + } + + assertEquals(false, resultColumn.getPrev(prevRowset.get(0))); + assertEquals(true, resultColumn.getPrev(prevRowset.get(1))); + assertEquals(false, resultColumn.getPrev(prevRowset.get(2))); + assertEquals(true, resultColumn.getPrev(prevRowset.get(3))); + + assertEquals(true, resultColumn.get(rs.get(0))); + assertEquals(true, resultColumn.get(rs.get(1))); + assertEquals(true, resultColumn.get(rs.get(2))); + assertEquals(false, resultColumn.get(rs.get(3))); + assertEquals(true, resultColumn.get(rs.get(4))); + }); + } + + @Test + public void testFilterExpressionFillChunkPerformance() { + testFilterExpressionFillChunkPerformance(1.0); + testFilterExpressionFillChunkPerformance(.9999); + testFilterExpressionFillChunkPerformance(.999); + testFilterExpressionFillChunkPerformance(.8725); + testFilterExpressionFillChunkPerformance(.75); + testFilterExpressionFillChunkPerformance(.5); + testFilterExpressionFillChunkPerformance(.25); + testFilterExpressionFillChunkPerformance(.125); + testFilterExpressionFillChunkPerformance(0.001); + testFilterExpressionFillChunkPerformance(0.0001); + } + + public void testFilterExpressionFillChunkPerformance(final double density) { + final int numIterations = 1; + final int size = 100_000; + final Filter filter = FilterIn.of(ColumnName.of("A"), Literal.of(1)); + + final Random random = new Random(20241120); + final List values = IntStream.range(0, size).mapToObj(ignored -> random.nextDouble() < density) + .collect(Collectors.toList()); + QueryScope.addParam("values", values); + final Table t = TableTools.emptyTable(size).update("A=(Boolean)(values[i]) ? 1: 0"); + QueryScope.addParam("values", null); + + final Table upv = t.updateView(List.of(Selectable.of(ColumnName.of("AWM"), filter))); + final long startTime = System.nanoTime(); + for (int iters = 0; iters < numIterations; ++iters) { + final long trueValues = upv.columnIterator("AWM").stream().filter(x -> (Boolean) x).count(); + assertEquals(values.stream().filter(x -> x).count(), trueValues); + } + final long endTime = System.nanoTime(); + final double duration = endTime - startTime; + System.out.println("Density: " + new DecimalFormat("0.0000").format(density) + ", Nanos: " + (long) duration + + ", per cell=" + new DecimalFormat("0.00").format(duration / (size * numIterations))); + } + + @Test + public void testFilterExpressionArray() { + final Filter filter = WhereFilterFactory.getExpression("A=A_[i-1]"); + final Filter filterArrayOnly = WhereFilterFactory.getExpression("A=A_.size() = 1"); + final Filter filterKonly = WhereFilterFactory.getExpression("A=k+1"); + final QueryTable setTable = TstUtils.testRefreshingTable(intCol("B")); + final Filter whereIn = new DynamicWhereFilter(setTable, true, MatchPairFactory.getExpression("A=B")); + final QueryTable table = TstUtils.testRefreshingTable(intCol("A", 1, 1, 2, 3, 5, 8, 9, 9)); + + final UncheckedTableException wme = Assert.assertThrows(UncheckedTableException.class, + () -> table.wouldMatch(new WouldMatchPair("AWM", filter))); + Assert.assertEquals("wouldMatch filters cannot use virtual row variables (i, ii, and k): [A=A_[i-1]]", + wme.getMessage()); + + final UncheckedTableException wme2 = Assert.assertThrows(UncheckedTableException.class, + () -> table.wouldMatch(new WouldMatchPair("AWM", filterArrayOnly))); + Assert.assertEquals("wouldMatch filters cannot use column Vectors (_ syntax): [A=A_.size() = 1]", + wme2.getMessage()); + + final UncheckedTableException upe = Assert.assertThrows(UncheckedTableException.class, + () -> table.update(List.of(Selectable.of(ColumnName.of("AWM"), filter)))); + Assert.assertEquals( + "Cannot use a filter with column Vectors (_ syntax) in select, view, update, or updateView: A=A_[i-1]", + upe.getMessage()); + + final UncheckedTableException uve = Assert.assertThrows(UncheckedTableException.class, + () -> table.updateView(List.of(Selectable.of(ColumnName.of("AWM"), filter)))); + Assert.assertEquals( + "Cannot use a filter with column Vectors (_ syntax) in select, view, update, or updateView: A=A_[i-1]", + uve.getMessage()); + + final UncheckedTableException se = Assert.assertThrows(UncheckedTableException.class, + () -> table.select(List.of(Selectable.of(ColumnName.of("AWM"), filterKonly)))); + Assert.assertEquals( + "Cannot use a filter with virtual row variables (i, ii, or k) in select, view, update, or updateView: A=k+1", + se.getMessage()); + + final UncheckedTableException ve = Assert.assertThrows(UncheckedTableException.class, + () -> table.view(List.of(Selectable.of(ColumnName.of("AWM"), filterKonly)))); + Assert.assertEquals( + "Cannot use a filter with virtual row variables (i, ii, or k) in select, view, update, or updateView: A=k+1", + ve.getMessage()); + + final UncheckedTableException dw = Assert.assertThrows(UncheckedTableException.class, + () -> table.view(List.of(Selectable.of(ColumnName.of("AWM"), whereIn)))); + Assert.assertEquals( + "Cannot use a refreshing filter in select, view, update, or updateView: DynamicWhereFilter([A=B])", + dw.getMessage()); + + } + + @Test + public void testFilterExpressionTicking() { + for (int seed = 0; seed < 5; ++seed) { + testFilterExpressionTicking(seed, new MutableInt(100)); + } + } + + private void testFilterExpressionTicking(final int seed, final MutableInt numSteps) { + final Random random = new Random(seed); + final ColumnInfo[] columnInfo; + final int size = 25; + final QueryTable queryTable = getTable(size, random, + columnInfo = initColumnInfos(new String[] {"Sym", "intCol", "doubleCol"}, + new SetGenerator<>("a", "b", "c", "d", "e"), + new IntGenerator(10, 100), + new SetGenerator<>(10.1, 20.1, 30.1))); + + final EvalNuggetInterface[] en = new EvalNuggetInterface[] { + new TableComparator(queryTable.wouldMatch("SM=Sym in `b`, `d`"), + queryTable.update(List.of(Selectable.of(ColumnName.of("SM"), + WhereFilterFactory.getExpression("Sym in `b`, `d`"))))), + new TableComparator(queryTable.wouldMatch("SM=Sym in `b`, `d`"), + queryTable.updateView(List.of(Selectable.of(ColumnName.of("SM"), + WhereFilterFactory.getExpression("Sym in `b`, `d`"))))), + new TableComparator(queryTable.wouldMatch("IM=intCol < 50"), + queryTable.update(List.of( + Selectable.of(ColumnName.of("IM"), WhereFilterFactory.getExpression("intCol < 50"))))), + new TableComparator(queryTable.wouldMatch("IM=intCol < 50"), + queryTable.updateView(List.of( + Selectable.of(ColumnName.of("IM"), WhereFilterFactory.getExpression("intCol < 50"))))), + new TableComparator(queryTable.wouldMatch("IM=Sym= (intCol%2 == 0? `a` : `b`)"), + queryTable.update(List.of(Selectable.of(ColumnName.of("IM"), + WhereFilterFactory.getExpression("Sym= (intCol%2 == 0? `a` : `b`)"))))), + new TableComparator(queryTable.wouldMatch("IM=Sym= (intCol%2 == 0? `a` : `b`)"), + queryTable.updateView(List.of(Selectable.of(ColumnName.of("IM"), + WhereFilterFactory.getExpression("Sym= (intCol%2 == 0? `a` : `b`)"))))), + }; + + final int maxSteps = numSteps.get(); + for (numSteps.set(0); numSteps.get() < maxSteps; numSteps.increment()) { + if (RefreshingTableTestCase.printTableUpdates) { + System.out.println("Step = " + numSteps.get()); + } + RefreshingTableTestCase.simulateShiftAwareStep(size, random, queryTable, columnInfo, en); + } + } + @Test public void testAlwaysUpdate() { final MutableInt count = new MutableInt(0);