From 36ad259734bbca6a0da5d7a8addf416e65ba4eda Mon Sep 17 00:00:00 2001 From: Alex Petrov Date: Tue, 3 Dec 2024 21:22:14 +0100 Subject: [PATCH] Move away from IndexedBijections --- .../test/log/CoordinatorPathTest.java | 12 +- .../fuzz/ring/ConsistentBootstrapTest.java | 4 +- .../fuzz/topology/HarryTopologyMixupTest.java | 2 +- .../org/apache/cassandra/harry/Relations.java | 13 +- .../apache/cassandra/harry/SchemaSpec.java | 3 +- .../cassandra/harry/ValueGeneratorHelper.java | 12 +- .../cassandra/harry/cql/DeleteHelper.java | 16 +- .../cassandra/harry/cql/SelectHelper.java | 14 +- .../cassandra/harry/cql/WriteHelper.java | 24 +-- .../cassandra/harry/dsl/HistoryBuilder.java | 179 +++++++++++++++++- .../harry/dsl/HistoryBuilderHelper.java | 18 +- .../harry/dsl/MultiOperationVisitBuilder.java | 5 +- .../harry/dsl/ReplayingHistoryBuilder.java | 4 +- .../dsl/SingleOperationVisitBuilder.java | 108 +++++------ .../execution/CQLTesterVisitExecutor.java | 8 +- .../execution/InJvmDTestVisitExecutor.java | 8 +- .../harry/execution/ResultSetRow.java | 29 ++- .../RingAwareInJvmDTestVisitExecutor.java | 2 +- .../cassandra/harry/gen/Bijections.java | 19 +- .../harry/gen/InvertibleGenerator.java | 3 +- .../harry/gen/OperationsGenerators.java | 24 ++- .../cassandra/harry/gen/ValueGenerators.java | 150 +++++++-------- .../cassandra/harry/model/PartitionState.java | 71 ++++--- .../harry/model/PartitionStateBuilder.java | 2 +- .../harry/test/HistoryBuilderTest.java | 7 +- .../harry/test/SimpleBijectionTest.java | 5 +- .../cassandra/harry/util/StringUtils.java | 11 ++ .../simulator/test/HarryValidatingQuery.java | 2 +- 28 files changed, 456 insertions(+), 299 deletions(-) diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTest.java index ce43dfd83661..42630b3a297c 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTest.java @@ -37,6 +37,7 @@ import org.apache.cassandra.harry.SchemaSpec; import org.apache.cassandra.harry.ValueGeneratorHelper; import org.apache.cassandra.harry.cql.WriteHelper; +import org.apache.cassandra.harry.dsl.HistoryBuilder; import org.apache.cassandra.harry.execution.CompiledStatement; import org.apache.cassandra.harry.gen.Generator; import org.apache.cassandra.harry.gen.SchemaGenerators; @@ -80,11 +81,12 @@ public void writeConsistencyTest() throws Throwable " WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};"); cluster.schemaChange(schema.compile()); - for (int i = 0; i < schema.valueGenerators.pkPopulation(); i++) + HistoryBuilder.IndexedValueGenerators valueGenerators = (HistoryBuilder.IndexedValueGenerators) schema.valueGenerators; + for (int i = 0; i < valueGenerators.pkPopulation(); i++) { - long pd = schema.valueGenerators.pkGen.descriptorAt(i); + long pd = valueGenerators.pkGen().descriptorAt(i); - ByteBuffer[] pk = ByteUtils.objectsToBytes(schema.valueGenerators.pkGen.inflate(pd)); + ByteBuffer[] pk = ByteUtils.objectsToBytes(valueGenerators.pkGen().inflate(pd)); long token = TokenUtil.token(ByteUtils.compose(pk)); if (!prediction.state.get().isWriteTargetFor(token, prediction.node(6).matcher)) continue; @@ -102,8 +104,8 @@ public void writeConsistencyTest() throws Throwable Future writeQuery = async(() -> { CompiledStatement s = WriteHelper.inflateInsert(new Operations.WriteOp(lts, pd, 0, - ValueGeneratorHelper.randomDescriptors(rng, schema.valueGenerators.regularColumnGens), - ValueGeneratorHelper.randomDescriptors(rng, schema.valueGenerators.staticColumnGens), + ValueGeneratorHelper.randomDescriptors(rng, valueGenerators::regularColumnGen, valueGenerators.regularColumnCount()), + ValueGeneratorHelper.randomDescriptors(rng, valueGenerators::staticColumnGen, valueGenerators.staticColumnCount()), Operations.Kind.INSERT), schema, lts); diff --git a/test/distributed/org/apache/cassandra/fuzz/ring/ConsistentBootstrapTest.java b/test/distributed/org/apache/cassandra/fuzz/ring/ConsistentBootstrapTest.java index 554b16f085fe..c6e8db78b18c 100644 --- a/test/distributed/org/apache/cassandra/fuzz/ring/ConsistentBootstrapTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/ring/ConsistentBootstrapTest.java @@ -212,9 +212,9 @@ public void coordinatorIsBehindTest() throws Throwable }, "Start grep"); outer: - for (int i = 0; i < schema.valueGenerators.pkPopulation(); i++) + for (int i = 0; i < history.valueGenerators().pkPopulation(); i++) { - long pd = schema.valueGenerators.pkGen.descriptorAt(i); + long pd = history.valueGenerators().pkGen().descriptorAt(i); for (TokenPlacementModel.Replica replica : executor.getReplicasFor(pd)) { if (cluster.get(1).config().broadcastAddress().toString().contains(replica.node().id())) diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/HarryTopologyMixupTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/HarryTopologyMixupTest.java index b8f9f54c3b29..f0d1b5650e5c 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/HarryTopologyMixupTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/HarryTopologyMixupTest.java @@ -231,7 +231,7 @@ private static CommandGen cqlOperations(Spec spec) for (Integer pkIdx : spec.pkGen.generated()) { - long pd = spec.schema.valueGenerators.pkGen.descriptorAt(pkIdx); + long pd = spec.harry.valueGenerators().pkGen().descriptorAt(pkIdx); reads.add(new HarryCommand(s -> String.format("Harry Validate pd=%d%s", pd, state.commandNamePostfix()), s -> spec.harry.selectPartition(pkIdx))); TransactionalMode transationalMode = spec.schema.options.transactionalMode(); diff --git a/test/harry/main/org/apache/cassandra/harry/Relations.java b/test/harry/main/org/apache/cassandra/harry/Relations.java index fafbba335265..91f64ded48b3 100644 --- a/test/harry/main/org/apache/cassandra/harry/Relations.java +++ b/test/harry/main/org/apache/cassandra/harry/Relations.java @@ -20,7 +20,7 @@ import java.util.Arrays; import java.util.Comparator; -import java.util.List; +import java.util.function.IntFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,8 +32,9 @@ public class Relations private static final Logger logger = LoggerFactory.getLogger(Relations.class); @SuppressWarnings({ "rawtypes", "unchecked" }) - public static boolean matchRange(Bijections.IndexedBijection ckGen, - List> comparators, + public static boolean matchRange(Bijections.Bijection ckGen, + IntFunction> comparators, + int ckCoulmnCount, long lowBoundDescr, long highBoundDescr, Relations.RelationKind[] lowBoundRelations, Relations.RelationKind[] highBoundRelations, long matchDescr) @@ -42,7 +43,7 @@ public static boolean matchRange(Bijections.IndexedBijection ckGen, Object[] highBoundValue = highBoundDescr == MagicConstants.UNSET_DESCR ? null : ckGen.inflate(highBoundDescr); Object[] matchValue = ckGen.inflate(matchDescr); // TODO: assert that all equals + null checks - for (int i = 0; i < comparators.size(); i++) + for (int i = 0; i < ckCoulmnCount; i++) { Object matched = matchValue[i]; @@ -51,7 +52,7 @@ public static boolean matchRange(Bijections.IndexedBijection ckGen, Object l = lowBoundValue[i]; Relations.RelationKind lr = lowBoundRelations[i]; - if (lr != null && !lr.match(comparators.get(i), matched, l)) + if (lr != null && !lr.match(comparators.apply(i), matched, l)) { if (logger.isTraceEnabled()) logger.trace("Low Bound {} {} {} did match {}", lowBoundValue[i], lr, matchValue[i], i); @@ -64,7 +65,7 @@ public static boolean matchRange(Bijections.IndexedBijection ckGen, Object h = highBoundValue[i]; Relations.RelationKind hr = highBoundRelations[i]; - if (hr != null && !hr.match(comparators.get(i), matched, h)) + if (hr != null && !hr.match(comparators.apply(i), matched, h)) { if (logger.isTraceEnabled()) logger.trace("High Bound {} {} {} did match {}", highBoundValue[i], hr, matchValue[i], i); diff --git a/test/harry/main/org/apache/cassandra/harry/SchemaSpec.java b/test/harry/main/org/apache/cassandra/harry/SchemaSpec.java index 84f5fbbe5e89..184c6d4e9439 100644 --- a/test/harry/main/org/apache/cassandra/harry/SchemaSpec.java +++ b/test/harry/main/org/apache/cassandra/harry/SchemaSpec.java @@ -25,6 +25,7 @@ import java.util.function.Consumer; import org.apache.cassandra.cql3.ast.Symbol; +import org.apache.cassandra.harry.dsl.HistoryBuilder; import org.apache.cassandra.harry.gen.Generator; import org.apache.cassandra.harry.gen.Generators; import org.apache.cassandra.harry.gen.ValueGenerators; @@ -94,7 +95,7 @@ public SchemaSpec(long seed, this.allColumnInSelectOrder = Collections.unmodifiableList(selectOrder); // TODO: empty gen - this.valueGenerators = ValueGenerators.fromSchema(this, seed, populationPerColumn); + this.valueGenerators = HistoryBuilder.valueGenerators(this, seed, populationPerColumn); } public static /* unsigned */ long cumulativeEntropy(List> columns) diff --git a/test/harry/main/org/apache/cassandra/harry/ValueGeneratorHelper.java b/test/harry/main/org/apache/cassandra/harry/ValueGeneratorHelper.java index ba1147bb4ca3..c4bbababc01b 100644 --- a/test/harry/main/org/apache/cassandra/harry/ValueGeneratorHelper.java +++ b/test/harry/main/org/apache/cassandra/harry/ValueGeneratorHelper.java @@ -18,22 +18,22 @@ package org.apache.cassandra.harry; -import java.util.List; +import java.util.function.IntFunction; -import org.apache.cassandra.harry.gen.Bijections; +import org.apache.cassandra.harry.dsl.HistoryBuilder; import org.apache.cassandra.harry.gen.EntropySource; public class ValueGeneratorHelper { - public static long[] randomDescriptors(EntropySource rng, List> valueGens) + public static long[] randomDescriptors(EntropySource rng, IntFunction> valueGens, int count) { - long[] vds = new long[valueGens.size()]; - for (int i = 0; i < valueGens.size(); i++) + long[] vds = new long[count]; + for (int i = 0; i < count; i++) { if (rng.nextBoolean()) vds[i] = MagicConstants.UNSET_DESCR; else - vds[i] = valueGens.get(i).descriptorAt(rng.nextInt(valueGens.size())); + vds[i] = valueGens.apply(i).descriptorAt(rng.nextInt(count)); } return vds; diff --git a/test/harry/main/org/apache/cassandra/harry/cql/DeleteHelper.java b/test/harry/main/org/apache/cassandra/harry/cql/DeleteHelper.java index 513728ef9fc3..1a237f58287a 100644 --- a/test/harry/main/org/apache/cassandra/harry/cql/DeleteHelper.java +++ b/test/harry/main/org/apache/cassandra/harry/cql/DeleteHelper.java @@ -50,7 +50,7 @@ public static CompiledStatement inflateDelete(Operations.DeletePartition delete, List bindings = new ArrayList<>(); - Object[] pk = schema.valueGenerators.pkGen.inflate(delete.pd()); + Object[] pk = schema.valueGenerators.pkGen().inflate(delete.pd()); RelationWriter writer = new RelationWriter(b, bindings::add) ; @@ -81,8 +81,8 @@ public static CompiledStatement inflateDelete(Operations.DeleteRow delete, List bindings = new ArrayList<>(); - Object[] pk = schema.valueGenerators.pkGen.inflate(delete.pd()); - Object[] ck = schema.valueGenerators.ckGen.inflate(delete.cd()); + Object[] pk = schema.valueGenerators.pkGen().inflate(delete.pd()); + Object[] ck = schema.valueGenerators.ckGen().inflate(delete.cd()); RelationWriter writer = new RelationWriter(b, bindings::add); @@ -139,8 +139,8 @@ public static CompiledStatement inflateDelete(Operations.DeleteColumns delete, List bindings = new ArrayList<>(); - Object[] pk = schema.valueGenerators.pkGen.inflate(delete.pd()); - Object[] ck = schema.valueGenerators.ckGen.inflate(delete.cd()); + Object[] pk = schema.valueGenerators.pkGen().inflate(delete.pd()); + Object[] ck = schema.valueGenerators.ckGen().inflate(delete.cd()); RelationWriter writer = new RelationWriter(b, bindings::add); @@ -173,9 +173,9 @@ public static CompiledStatement inflateDelete(Operations.DeleteRange delete, List bindings = new ArrayList<>(); - Object[] pk = schema.valueGenerators.pkGen.inflate(delete.pd()); - Object[] lowBound = schema.valueGenerators.ckGen.inflate(delete.lowerBound()); - Object[] highBound = schema.valueGenerators.ckGen.inflate(delete.upperBound()); + Object[] pk = schema.valueGenerators.pkGen().inflate(delete.pd()); + Object[] lowBound = schema.valueGenerators.ckGen().inflate(delete.lowerBound()); + Object[] highBound = schema.valueGenerators.ckGen().inflate(delete.upperBound()); RelationWriter writer = new RelationWriter(b, bindings::add); diff --git a/test/harry/main/org/apache/cassandra/harry/cql/SelectHelper.java b/test/harry/main/org/apache/cassandra/harry/cql/SelectHelper.java index 1cbbeb6a2224..f9d607fb6531 100644 --- a/test/harry/main/org/apache/cassandra/harry/cql/SelectHelper.java +++ b/test/harry/main/org/apache/cassandra/harry/cql/SelectHelper.java @@ -55,7 +55,7 @@ public static CompiledStatement select(Operations.SelectRow select, SchemaSpec s { Select.Builder builder = commmonPart(select, schema); - Object[] ck = schema.valueGenerators.ckGen.inflate(select.cd()); + Object[] ck = schema.valueGenerators.ckGen().inflate(select.cd()); for (int i = 0; i < schema.clusteringKeys.size(); i++) { @@ -72,8 +72,8 @@ public static CompiledStatement select(Operations.SelectRange select, SchemaSpec { Select.Builder builder = commmonPart(select, schema); - Object[] lowBound = schema.valueGenerators.ckGen.inflate(select.lowerBound()); - Object[] highBound = schema.valueGenerators.ckGen.inflate(select.upperBound()); + Object[] lowBound = schema.valueGenerators.ckGen().inflate(select.lowerBound()); + Object[] highBound = schema.valueGenerators.ckGen().inflate(select.upperBound()); for (int i = 0; i < schema.clusteringKeys.size(); i++) { @@ -112,7 +112,7 @@ public static CompiledStatement select(Operations.SelectCustom select, SchemaSpe Map cache = new HashMap<>(); for (Relations.Relation relation : select.ckRelations()) { - Object[] query = cache.computeIfAbsent(relation.descriptor, schema.valueGenerators.ckGen::inflate); + Object[] query = cache.computeIfAbsent(relation.descriptor, schema.valueGenerators.ckGen()::inflate); ColumnSpec column = schema.clusteringKeys.get(relation.column); builder.withWhere(Reference.of(new Symbol(column.name, column.type.asServerType())), toInequalities(relation.kind), @@ -122,7 +122,7 @@ public static CompiledStatement select(Operations.SelectCustom select, SchemaSpe for (Relations.Relation relation : select.regularRelations()) { ColumnSpec column = schema.regularColumns.get(relation.column); - Object query = schema.valueGenerators.regularColumnGens.get(relation.column).inflate(relation.descriptor); + Object query = schema.valueGenerators.regularColumnGen(relation.column).inflate(relation.descriptor); builder.withWhere(Reference.of(new Symbol(column.name, column.type.asServerType())), toInequalities(relation.kind), new Bind(query, column.type.asServerType())); @@ -130,7 +130,7 @@ public static CompiledStatement select(Operations.SelectCustom select, SchemaSpe for (Relations.Relation relation : select.staticRelations()) { - Object query = schema.valueGenerators.staticColumnGens.get(relation.column).inflate(relation.descriptor); + Object query = schema.valueGenerators.staticColumnGen(relation.column).inflate(relation.descriptor); ColumnSpec column = schema.staticColumns.get(relation.column); builder.withWhere(Reference.of(new Symbol(column.name, column.type.asServerType())), toInequalities(relation.kind), @@ -191,7 +191,7 @@ public static Select.Builder commmonPart(Operations.SelectStatement select, Sche builder.withTable(schema.keyspace, schema.table); - Object[] pk = schema.valueGenerators.pkGen.inflate(select.pd()); + Object[] pk = schema.valueGenerators.pkGen().inflate(select.pd()); for (int i = 0; i < schema.partitionKeys.size(); i++) { ColumnSpec column = schema.partitionKeys.get(i); diff --git a/test/harry/main/org/apache/cassandra/harry/cql/WriteHelper.java b/test/harry/main/org/apache/cassandra/harry/cql/WriteHelper.java index fa37a97bed43..b3143a7043f4 100644 --- a/test/harry/main/org/apache/cassandra/harry/cql/WriteHelper.java +++ b/test/harry/main/org/apache/cassandra/harry/cql/WriteHelper.java @@ -35,12 +35,12 @@ public static CompiledStatement inflateInsert(Operations.WriteOp op, { assert op.vds().length == schema.regularColumns.size(); assert op.sds().length == schema.staticColumns.size(); - assert op.vds().length == schema.valueGenerators.regularColumnGens.size(); - assert op.sds().length == schema.valueGenerators.staticColumnGens.size(); + assert op.vds().length == schema.valueGenerators.regularColumnCount(); + assert op.sds().length == schema.valueGenerators.staticColumnCount(); - Object[] partitionKey = schema.valueGenerators.pkGen.inflate(op.pd()); + Object[] partitionKey = schema.valueGenerators.pkGen().inflate(op.pd()); assert partitionKey.length == schema.partitionKeys.size(); - Object[] clusteringKey = schema.valueGenerators.ckGen.inflate(op.cd()); + Object[] clusteringKey = schema.valueGenerators.ckGen().inflate(op.cd()); assert clusteringKey.length == schema.clusteringKeys.size(); Object[] regularColumns = new Object[op.vds().length]; Object[] staticColumns = new Object[op.sds().length]; @@ -51,7 +51,7 @@ public static CompiledStatement inflateInsert(Operations.WriteOp op, if (descriptor == MagicConstants.UNSET_DESCR) regularColumns[i] = MagicConstants.UNSET_VALUE; else - regularColumns[i] = schema.valueGenerators.regularColumnGens.get(i).inflate(descriptor); + regularColumns[i] = schema.valueGenerators.regularColumnGen(i).inflate(descriptor); } for (int i = 0; i < op.sds().length; i++) @@ -60,7 +60,7 @@ public static CompiledStatement inflateInsert(Operations.WriteOp op, if (descriptor == MagicConstants.UNSET_DESCR) staticColumns[i] = MagicConstants.UNSET_VALUE; else - staticColumns[i] = schema.valueGenerators.staticColumnGens.get(i).inflate(descriptor); + staticColumns[i] = schema.valueGenerators.staticColumnGen(i).inflate(descriptor); } Object[] bindings = new Object[schema.allColumnInSelectOrder.size()]; @@ -115,21 +115,21 @@ public static CompiledStatement inflateUpdate(Operations.WriteOp op, { assert op.vds().length == schema.regularColumns.size(); assert op.sds().length == schema.staticColumns.size(); - assert op.vds().length == schema.valueGenerators.regularColumnGens.size(); - assert op.sds().length == schema.valueGenerators.staticColumnGens.size(); + assert op.vds().length == schema.valueGenerators.regularColumnCount(); + assert op.sds().length == schema.valueGenerators.staticColumnCount(); - Object[] partitionKey = schema.valueGenerators.pkGen.inflate(op.pd); + Object[] partitionKey = schema.valueGenerators.pkGen().inflate(op.pd); assert partitionKey.length == schema.partitionKeys.size(); - Object[] clusteringKey = schema.valueGenerators.ckGen.inflate(op.cd()); + Object[] clusteringKey = schema.valueGenerators.ckGen().inflate(op.cd()); assert clusteringKey.length == schema.clusteringKeys.size(); Object[] regularColumns = new Object[op.vds().length]; Object[] staticColumns = new Object[op.sds().length]; for (int i = 0; i < op.vds().length; i++) - regularColumns[i] = schema.valueGenerators.regularColumnGens.get(i).inflate(op.vds()[i]); + regularColumns[i] = schema.valueGenerators.regularColumnGen(i).inflate(op.vds()[i]); for (int i = 0; i < op.sds().length; i++) - staticColumns[i] = schema.valueGenerators.staticColumnGens.get(i).inflate(op.sds()[i]); + staticColumns[i] = schema.valueGenerators.staticColumnGen(i).inflate(op.sds()[i]); Object[] bindings = new Object[schema.allColumnInSelectOrder.size()]; diff --git a/test/harry/main/org/apache/cassandra/harry/dsl/HistoryBuilder.java b/test/harry/main/org/apache/cassandra/harry/dsl/HistoryBuilder.java index 3f9f954347c4..35631d55a0fa 100644 --- a/test/harry/main/org/apache/cassandra/harry/dsl/HistoryBuilder.java +++ b/test/harry/main/org/apache/cassandra/harry/dsl/HistoryBuilder.java @@ -18,21 +18,38 @@ package org.apache.cassandra.harry.dsl; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import org.apache.cassandra.harry.ColumnSpec; +import org.apache.cassandra.harry.MagicConstants; +import org.apache.cassandra.harry.SchemaSpec; +import org.apache.cassandra.harry.gen.Bijections; +import org.apache.cassandra.harry.gen.EntropySource; import org.apache.cassandra.harry.gen.IndexGenerators; +import org.apache.cassandra.harry.gen.InvertibleGenerator; import org.apache.cassandra.harry.gen.ValueGenerators; +import org.apache.cassandra.harry.gen.rng.JdkRandomEntropySource; import org.apache.cassandra.harry.op.Operations; import org.apache.cassandra.harry.op.Visit; import org.apache.cassandra.harry.model.Model; import org.apache.cassandra.harry.util.BitSet; +import org.apache.cassandra.harry.util.IteratorsUtil; + +import static org.apache.cassandra.harry.SchemaSpec.cumulativeEntropy; +import static org.apache.cassandra.harry.SchemaSpec.forKeys; +import static org.apache.cassandra.harry.gen.InvertibleGenerator.fromType; // TODO: either create or replay timestamps out of order public class HistoryBuilder implements SingleOperationBuilder, Model.Replay { - protected final ValueGenerators valueGenerators; + protected final IndexedValueGenerators valueGenerators; protected final IndexGenerators indexGenerators; protected int nextOpIdx = 0; @@ -40,12 +57,18 @@ public class HistoryBuilder implements SingleOperationBuilder, Model.Replay // TODO: would be great to have a very simple B-Tree here protected final Map log; - public HistoryBuilder(ValueGenerators valueGenerators) + public static HistoryBuilder fromSchema(SchemaSpec schemaSpec, long seed, int population) + { + IndexedValueGenerators generators = valueGenerators(schemaSpec, seed, population); + return new HistoryBuilder(generators, IndexGenerators.withDefaults(generators)); + } + + public HistoryBuilder(ValueGenerators generators) { - this(valueGenerators, IndexGenerators.withDefaults(valueGenerators)); + this((IndexedValueGenerators) generators, IndexGenerators.withDefaults(generators)); } - public HistoryBuilder(ValueGenerators valueGenerators, + public HistoryBuilder(IndexedValueGenerators valueGenerators, IndexGenerators indexGenerators) { this.log = new HashMap<>(); @@ -53,6 +76,11 @@ public HistoryBuilder(ValueGenerators valueGenerators, this.indexGenerators = indexGenerators; } + public IndexedValueGenerators valueGenerators() + { + return valueGenerators; + } + public int size() { return log.size(); @@ -97,7 +125,9 @@ SingleOperationVisitBuilder singleOpVisitBuilder() indexGenerators, (visit) -> log.put(visit.lts, visit)); } -; + + ; + public MultiOperationVisitBuilder multistep() { long visitLts = nextOpIdx++; @@ -271,4 +301,141 @@ public SingleOperationBuilder deleteRowSliceByUpperBound(int pdIdx, int upperBou singleOpVisitBuilder().deleteRowSliceByUpperBound(pdIdx, upperBoundRowIdx, nonEqFrom, isEq); return this; } -} + + + /** + * Indexed bijection allows to decouple descriptor order from value order, which makes data generation simpler. + *

+ * For regular Harry bijections, this is done at no cost, since values are inflated in a way that preserves + * descriptor order, which means that idx order is consistent with descriptor order and consistent with value order. + *

+ * An indexed bijection allows order to be established via index, and use descriptor simply as a seed for random values. + */ + public interface IndexedBijection extends Bijections.Bijection + { + int idxFor(long descriptor); + + long descriptorAt(int idx); + + @Override + default String toString(long pd) + { + if (pd == MagicConstants.UNSET_DESCR) + return Integer.toString(MagicConstants.UNSET_IDX); + + if (pd == MagicConstants.NIL_DESCR) + return Integer.toString(MagicConstants.NIL_IDX); + + return Integer.toString(idxFor(pd)); + } + } + + public static IndexedValueGenerators valueGenerators(SchemaSpec schema, long seed) + { + return valueGenerators(schema, seed, 1000); + } + + @SuppressWarnings({ "unchecked" }) + public static IndexedValueGenerators valueGenerators(SchemaSpec schema, long seed, int populationPerColumn) + { + List> pkComparators = new ArrayList<>(); + List> ckComparators = new ArrayList<>(); + List> regularComparators = new ArrayList<>(); + List> staticComparators = new ArrayList<>(); + + EntropySource rng = new JdkRandomEntropySource(seed); + for (int i = 0; i < schema.partitionKeys.size(); i++) + pkComparators.add((Comparator) schema.partitionKeys.get(i).type.comparator()); + for (int i = 0; i < schema.clusteringKeys.size(); i++) + ckComparators.add((Comparator) schema.clusteringKeys.get(i).type.comparator()); + for (int i = 0; i < schema.regularColumns.size(); i++) + regularComparators.add((Comparator) schema.regularColumns.get(i).type.comparator()); + for (int i = 0; i < schema.staticColumns.size(); i++) + staticComparators.add((Comparator) schema.staticColumns.get(i).type.comparator()); + + Map, InvertibleGenerator> map = new HashMap<>(); + for (ColumnSpec column : IteratorsUtil.concat(schema.regularColumns, schema.staticColumns)) + map.computeIfAbsent(column, (a) -> (InvertibleGenerator) fromType(rng, populationPerColumn, column)); + + // TODO: empty gen + return new IndexedValueGenerators(new InvertibleGenerator<>(rng, cumulativeEntropy(schema.partitionKeys), populationPerColumn, forKeys(schema.partitionKeys), keyComparator(schema.partitionKeys)), + new InvertibleGenerator<>(rng, cumulativeEntropy(schema.clusteringKeys), populationPerColumn, forKeys(schema.clusteringKeys), keyComparator(schema.clusteringKeys)), + schema.regularColumns.stream() + .map(map::get) + .collect(Collectors.toList()), + schema.staticColumns.stream() + .map(map::get) + .collect(Collectors.toList()), + pkComparators, + ckComparators, + regularComparators, + staticComparators); + } + + public static class IndexedValueGenerators extends ValueGenerators + { + public IndexedValueGenerators(IndexedBijection pkGen, + IndexedBijection ckGen, + List> regularColumnGens, + List> staticColumnGens, + List> pkComparators, + List> ckComparators, + List> regularComparators, + List> staticComparators) + { + super(pkGen, ckGen, + (List>) (List) regularColumnGens, + (List>) (List) staticColumnGens, + pkComparators, ckComparators, regularComparators, staticComparators); + } + + @Override + public IndexedBijection pkGen() + { + return (IndexedBijection) super.pkGen(); + } + + @Override + public IndexedBijection ckGen() + { + return (IndexedBijection) super.ckGen(); + } + + @Override + public IndexedBijection regularColumnGen(int idx) + { + return (IndexedBijection) super.regularColumnGen(idx); + } + + @Override + public IndexedBijection staticColumnGen(int idx) + { + return (IndexedBijection) super.staticColumnGen(idx); + } + } + + + private static Comparator keyComparator(List> columns) + { + return (o1, o2) -> compareKeys(columns, o1, o2); + } + + public static int compareKeys(List> columns, Object[] v1, Object[] v2) + { + assert v1.length == v2.length : String.format("Values should be of same length: %d != %d\n%s\n%s", + v1.length, v2.length, Arrays.toString(v1), Arrays.toString(v2)); + + for (int i = 0; i < v1.length; i++) + { + int res; + ColumnSpec column = columns.get(i); + if (column.type.isReversed()) + res = column.type.comparator().reversed().compare(v1[i], v2[i]); + else + res = column.type.comparator().compare(v1[i], v2[i]); + if (res != 0) + return res; + } + return 0; + } +} \ No newline at end of file diff --git a/test/harry/main/org/apache/cassandra/harry/dsl/HistoryBuilderHelper.java b/test/harry/main/org/apache/cassandra/harry/dsl/HistoryBuilderHelper.java index e255ec1ee114..281f62cc87f6 100644 --- a/test/harry/main/org/apache/cassandra/harry/dsl/HistoryBuilderHelper.java +++ b/test/harry/main/org/apache/cassandra/harry/dsl/HistoryBuilderHelper.java @@ -47,40 +47,40 @@ public class HistoryBuilderHelper /** * Perform a random insert to any row */ - public static void insertRandomData(SchemaSpec schema, Generator pkGen, Generator ckGen, EntropySource rng, SingleOperationBuilder history) + public static void insertRandomData(SchemaSpec schema, Generator pkGen, Generator ckGen, EntropySource rng, HistoryBuilder history) { insertRandomData(schema, pkGen.generate(rng), ckGen.generate(rng), rng, history); } - public static void insertRandomData(SchemaSpec schema, int partitionIdx, int rowIdx, EntropySource rng, SingleOperationBuilder history) + public static void insertRandomData(SchemaSpec schema, int partitionIdx, int rowIdx, EntropySource rng, HistoryBuilder history) { int[] vIdxs = new int[schema.regularColumns.size()]; for (int i = 0; i < schema.regularColumns.size(); i++) - vIdxs[i] = rng.nextInt(schema.valueGenerators.regularPopulation(i)); + vIdxs[i] = rng.nextInt(history.valueGenerators().regularPopulation(i)); int[] sIdxs = new int[schema.staticColumns.size()]; for (int i = 0; i < schema.staticColumns.size(); i++) - sIdxs[i] = rng.nextInt(schema.valueGenerators.staticPopulation(i)); + sIdxs[i] = rng.nextInt(history.valueGenerators().staticPopulation(i)); history.insert(partitionIdx, rowIdx, vIdxs, sIdxs); } - public static void insertRandomData(SchemaSpec schema, int pkIdx, EntropySource rng, SingleOperationBuilder history) + public static void insertRandomData(SchemaSpec schema, int pkIdx, EntropySource rng, HistoryBuilder history) { insertRandomData(schema, pkIdx, - rng.nextInt(0, schema.valueGenerators.ckPopulation()), + rng.nextInt(0, history.valueGenerators().ckPopulation()), rng, 0, history); } - public static void insertRandomData(SchemaSpec schema, int partitionIdx, int rowIdx, EntropySource rng, double chanceOfUnset, SingleOperationBuilder history) + public static void insertRandomData(SchemaSpec schema, int partitionIdx, int rowIdx, EntropySource rng, double chanceOfUnset, HistoryBuilder history) { int[] vIdxs = new int[schema.regularColumns.size()]; for (int i = 0; i < schema.regularColumns.size(); i++) - vIdxs[i] = rng.nextDouble() <= chanceOfUnset ? MagicConstants.UNSET_IDX : rng.nextInt(schema.valueGenerators.regularPopulation(i)); + vIdxs[i] = rng.nextDouble() <= chanceOfUnset ? MagicConstants.UNSET_IDX : rng.nextInt(history.valueGenerators().regularPopulation(i)); int[] sIdxs = new int[schema.staticColumns.size()]; for (int i = 0; i < schema.staticColumns.size(); i++) - sIdxs[i] = rng.nextDouble() <= chanceOfUnset ? MagicConstants.UNSET_IDX : rng.nextInt(schema.valueGenerators.staticPopulation(i)); + sIdxs[i] = rng.nextDouble() <= chanceOfUnset ? MagicConstants.UNSET_IDX : rng.nextInt(history.valueGenerators().staticPopulation(i)); history.insert(partitionIdx, rowIdx, vIdxs, sIdxs); } diff --git a/test/harry/main/org/apache/cassandra/harry/dsl/MultiOperationVisitBuilder.java b/test/harry/main/org/apache/cassandra/harry/dsl/MultiOperationVisitBuilder.java index 92c0eac8e617..7dbff3db2e14 100644 --- a/test/harry/main/org/apache/cassandra/harry/dsl/MultiOperationVisitBuilder.java +++ b/test/harry/main/org/apache/cassandra/harry/dsl/MultiOperationVisitBuilder.java @@ -22,13 +22,14 @@ import java.util.function.Consumer; import org.apache.cassandra.harry.gen.IndexGenerators; -import org.apache.cassandra.harry.gen.ValueGenerators; import org.apache.cassandra.harry.op.Visit; import org.apache.cassandra.harry.util.BitSet; +import static org.apache.cassandra.harry.dsl.HistoryBuilder.IndexedValueGenerators; + public class MultiOperationVisitBuilder extends SingleOperationVisitBuilder implements Closeable { - MultiOperationVisitBuilder(long lts, ValueGenerators valueGenerators, IndexGenerators indexGenerators, Consumer appendToLog) + MultiOperationVisitBuilder(long lts, IndexedValueGenerators valueGenerators, IndexGenerators indexGenerators, Consumer appendToLog) { super(lts, valueGenerators, indexGenerators, appendToLog); } diff --git a/test/harry/main/org/apache/cassandra/harry/dsl/ReplayingHistoryBuilder.java b/test/harry/main/org/apache/cassandra/harry/dsl/ReplayingHistoryBuilder.java index b923f04bd076..830c3ad4efcd 100644 --- a/test/harry/main/org/apache/cassandra/harry/dsl/ReplayingHistoryBuilder.java +++ b/test/harry/main/org/apache/cassandra/harry/dsl/ReplayingHistoryBuilder.java @@ -28,9 +28,9 @@ public class ReplayingHistoryBuilder extends HistoryBuilder { private final CQLVisitExecutor executor; - public ReplayingHistoryBuilder(ValueGenerators valueGenerators, Function executorFactory) + public ReplayingHistoryBuilder(ValueGenerators generators, Function executorFactory) { - super(valueGenerators); + super((IndexedValueGenerators) generators); this.executor = executorFactory.apply(this); } diff --git a/test/harry/main/org/apache/cassandra/harry/dsl/SingleOperationVisitBuilder.java b/test/harry/main/org/apache/cassandra/harry/dsl/SingleOperationVisitBuilder.java index e1906f496585..b4f2ff908bf2 100644 --- a/test/harry/main/org/apache/cassandra/harry/dsl/SingleOperationVisitBuilder.java +++ b/test/harry/main/org/apache/cassandra/harry/dsl/SingleOperationVisitBuilder.java @@ -33,11 +33,11 @@ import org.apache.cassandra.harry.gen.rng.SeedableEntropySource; import org.apache.cassandra.harry.MagicConstants; import org.apache.cassandra.harry.Relations; -import org.apache.cassandra.harry.gen.ValueGenerators; import org.apache.cassandra.harry.op.Operations; import org.apache.cassandra.harry.op.Visit; import org.apache.cassandra.harry.util.BitSet; +import static org.apache.cassandra.harry.dsl.HistoryBuilder.*; import static org.apache.cassandra.harry.op.Operations.Kind; import static org.apache.cassandra.harry.op.Operations.Operation; import static org.apache.cassandra.harry.op.Operations.WriteOp; @@ -57,11 +57,11 @@ class SingleOperationVisitBuilder implements SingleOperationBuilder protected int opIdCounter; - protected final ValueGenerators valueGenerators; + protected final IndexedValueGenerators valueGenerators; protected final IndexGenerators indexGenerators; SingleOperationVisitBuilder(long lts, - ValueGenerators valueGenerators, + IndexedValueGenerators valueGenerators, IndexGenerators indexGenerators, Consumer appendToLog) { @@ -220,21 +220,21 @@ public SingleOperationBuilder update(int pdIdx, int cdIdx, int[] valueIdxs, int[ private SingleOperationBuilder write(int pdIdx, int cdIdx, int[] valueIdxs, int[] sValueIdxs, Kind kind) { - assert valueIdxs.length == valueGenerators.regularColumnGens.size(); - assert sValueIdxs.length == valueGenerators.staticColumnGens.size(); + assert valueIdxs.length == valueGenerators.regularColumnCount(); + assert sValueIdxs.length == valueGenerators.staticColumnCount(); - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); - long cd = valueGenerators.ckGen.descriptorAt(cdIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); + long cd = valueGenerators.ckGen().descriptorAt(cdIdx); opIdCounter++; long[] vds = new long[valueIdxs.length]; - for (int i = 0; i < valueGenerators.regularColumnGens.size(); i++) + for (int i = 0; i < valueGenerators.regularColumnCount(); i++) { int valueIdx = valueIdxs[i]; if (valueIdx == MagicConstants.UNSET_IDX) vds[i] = MagicConstants.UNSET_DESCR; else - vds[i] = valueGenerators.regularColumnGens.get(i).descriptorAt(valueIdx); + vds[i] = valueGenerators.regularColumnGen(i).descriptorAt(valueIdx); } long[] sds = new long[sValueIdxs.length]; @@ -244,7 +244,7 @@ private SingleOperationBuilder write(int pdIdx, int cdIdx, int[] valueIdxs, int[ if (valueIdx == MagicConstants.UNSET_IDX) sds[i] = MagicConstants.UNSET_DESCR; else - sds[i] = valueGenerators.staticColumnGens.get(i).descriptorAt(valueIdx); + sds[i] = valueGenerators.staticColumnGen(i).descriptorAt(valueIdx); } operations.add(new WriteOp(lts, pd, cd, vds, sds, kind) { @@ -263,17 +263,17 @@ public String toString() public SingleOperationVisitBuilder deleteRowRange(int pdIdx, int lowerBoundRowIdx, int upperBoundRowIdx, int nonEqFrom, boolean includeLowerBound, boolean includeUpperBound) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); - long lowerBoundCd = valueGenerators.ckGen.descriptorAt(lowerBoundRowIdx); - long upperBoundCd = valueGenerators.ckGen.descriptorAt(upperBoundRowIdx); + long lowerBoundCd = valueGenerators.ckGen().descriptorAt(lowerBoundRowIdx); + long upperBoundCd = valueGenerators.ckGen().descriptorAt(upperBoundRowIdx); - Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[valueGenerators.ckComparators.size()]; - Relations.RelationKind[] upperBoundRelations = new Relations.RelationKind[valueGenerators.ckComparators.size()]; + Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()]; + Relations.RelationKind[] upperBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()]; int opId = opIdCounter++; rngSupplier.doWithSeed(opId, rng -> { - for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckComparators.size()); i++) + for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckColumnCount()); i++) { if (i < nonEqFrom) lowerBoundRelations[i] = Relations.RelationKind.EQ; @@ -305,7 +305,7 @@ public String toString() @Override public SingleOperationBuilder deletePartition(int pdIdx) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); opIdCounter++; operations.add(new Operations.DeletePartition(lts, pd) { @@ -322,9 +322,9 @@ public String toString() @Override public SingleOperationBuilder deleteRow(int pdIdx, int rowIdx) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); opIdCounter++; - long cd = valueGenerators.ckGen.descriptorAt(rowIdx); + long cd = valueGenerators.ckGen().descriptorAt(rowIdx); operations.add(new Operations.DeleteRow(lts, pd, cd) { @Override public String toString() @@ -339,9 +339,9 @@ public String toString() @Override public SingleOperationBuilder deleteColumns(int pdIdx, int rowIdx, BitSet regularSelection, BitSet staticSelection) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); opIdCounter++; - long cd = valueGenerators.ckGen.descriptorAt(rowIdx); + long cd = valueGenerators.ckGen().descriptorAt(rowIdx); operations.add(new Operations.DeleteColumns(lts, pd, cd, regularSelection, staticSelection) { @Override public String toString() @@ -356,14 +356,14 @@ public String toString() @Override public SingleOperationBuilder deleteRowSliceByLowerBound(int pdIdx, int lowerBoundRowIdx, int nonEqFrom, boolean includeBound) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); - long lowerBoundCd = valueGenerators.ckGen.descriptorAt(lowerBoundRowIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); + long lowerBoundCd = valueGenerators.ckGen().descriptorAt(lowerBoundRowIdx); - Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[valueGenerators.ckComparators.size()]; + Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()]; int opId = opIdCounter++; rngSupplier.doWithSeed(opId, rng -> { - for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckComparators.size()); i++) + for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckColumnCount()); i++) { if (i < nonEqFrom) lowerBoundRelations[i] = Relations.RelationKind.EQ; @@ -389,14 +389,14 @@ public String toString() @Override public SingleOperationBuilder deleteRowSliceByUpperBound(int pdIdx, int upperBoundRowIdx, int nonEqFrom, boolean includeBound) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); - long upperBoundCd = valueGenerators.ckGen.descriptorAt(upperBoundRowIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); + long upperBoundCd = valueGenerators.ckGen().descriptorAt(upperBoundRowIdx); - Relations.RelationKind[] upperBoundRelations = new Relations.RelationKind[valueGenerators.ckComparators.size()]; + Relations.RelationKind[] upperBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()]; int opId = opIdCounter++; rngSupplier.doWithSeed(opId, rng -> { - for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckComparators.size()); i++) + for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckColumnCount()); i++) { if (i < nonEqFrom) upperBoundRelations[i] = Relations.RelationKind.EQ; @@ -424,16 +424,16 @@ public String toString() public SingleOperationVisitBuilder selectRowRange(int pdIdx, int lowerBoundRowIdx, int upperBoundRowIdx, int nonEqFrom, boolean includeLowerBound, boolean includeUpperBound) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); - long lowerBoundCd = valueGenerators.ckGen.descriptorAt(lowerBoundRowIdx); - long upperBoundCd = valueGenerators.ckGen.descriptorAt(upperBoundRowIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); + long lowerBoundCd = valueGenerators.ckGen().descriptorAt(lowerBoundRowIdx); + long upperBoundCd = valueGenerators.ckGen().descriptorAt(upperBoundRowIdx); - Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[valueGenerators.ckComparators.size()]; - Relations.RelationKind[] upperBoundRelations = new Relations.RelationKind[valueGenerators.ckComparators.size()]; + Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()]; + Relations.RelationKind[] upperBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()]; int opId = opIdCounter++; rngSupplier.doWithSeed(opId, rng -> { - for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckComparators.size()); i++) + for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckColumnCount()); i++) { if (i < nonEqFrom) lowerBoundRelations[i] = Relations.RelationKind.EQ; @@ -455,33 +455,33 @@ public SingleOperationVisitBuilder selectRowRange(int pdIdx, int lowerBoundRowId @Override public SingleOperationBuilder select(int pdIdx, IdxRelation[] ckIdxRelations, IdxRelation[] regularIdxRelations, IdxRelation[] staticIdxRelations) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); opIdCounter++; Relations.Relation[] ckRelations = new Relations.Relation[ckIdxRelations.length]; for (int i = 0; i < ckRelations.length; i++) { - Invariants.checkState(ckIdxRelations[i].column < valueGenerators.ckComparators.size()); + Invariants.checkState(ckIdxRelations[i].column < valueGenerators.ckColumnCount()); ckRelations[i] = new Relations.Relation(ckIdxRelations[i].kind, - valueGenerators.ckGen.descriptorAt(ckIdxRelations[i].idx), + valueGenerators.ckGen().descriptorAt(ckIdxRelations[i].idx), ckIdxRelations[i].column); } Relations.Relation[] regularRelations = new Relations.Relation[regularIdxRelations.length]; for (int i = 0; i < regularRelations.length; i++) { - Invariants.checkState(regularIdxRelations[i].column < valueGenerators.regularComparators.size()); + Invariants.checkState(regularIdxRelations[i].column < valueGenerators.regularColumnCount()); regularRelations[i] = new Relations.Relation(regularIdxRelations[i].kind, - valueGenerators.regularColumnGens.get(regularIdxRelations[i].column).descriptorAt(regularIdxRelations[i].idx), + valueGenerators.regularColumnGen(regularIdxRelations[i].column).descriptorAt(regularIdxRelations[i].idx), regularIdxRelations[i].column); } Relations.Relation[] staticRelations = new Relations.Relation[staticIdxRelations.length]; for (int i = 0; i < staticRelations.length; i++) { - Invariants.checkState(staticIdxRelations[i].column < valueGenerators.staticComparators.size()); + Invariants.checkState(staticIdxRelations[i].column < valueGenerators.staticColumnCount()); staticRelations[i] = new Relations.Relation(staticIdxRelations[i].kind, - valueGenerators.staticColumnGens.get(staticIdxRelations[i].column).descriptorAt(staticIdxRelations[i].idx), + valueGenerators.staticColumnGen(staticIdxRelations[i].column).descriptorAt(staticIdxRelations[i].idx), staticIdxRelations[i].column); } @@ -493,7 +493,7 @@ public SingleOperationBuilder select(int pdIdx, IdxRelation[] ckIdxRelations, Id @Override public SingleOperationBuilder selectPartition(int pdIdx) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); opIdCounter++; operations.add(new Operations.SelectPartition(lts, pd)); @@ -504,7 +504,7 @@ public SingleOperationBuilder selectPartition(int pdIdx) @Override public SingleOperationBuilder selectPartition(int pdIdx, Operations.ClusteringOrderBy orderBy) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); opIdCounter++; operations.add(new Operations.SelectPartition(lts, pd, orderBy)); @@ -515,9 +515,9 @@ public SingleOperationBuilder selectPartition(int pdIdx, Operations.ClusteringOr @Override public SingleOperationBuilder selectRow(int pdIdx, int rowIdx) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); opIdCounter++; - long cd = valueGenerators.ckGen.descriptorAt(rowIdx); + long cd = valueGenerators.ckGen().descriptorAt(rowIdx); operations.add(new Operations.SelectRow(lts, pd, cd)); build(); return this; @@ -526,14 +526,14 @@ public SingleOperationBuilder selectRow(int pdIdx, int rowIdx) @Override public SingleOperationBuilder selectRowSliceByLowerBound(int pdIdx, int lowerBoundRowIdx, int nonEqFrom, boolean includeBound) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); - long lowerBoundCd = valueGenerators.ckGen.descriptorAt(lowerBoundRowIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); + long lowerBoundCd = valueGenerators.ckGen().descriptorAt(lowerBoundRowIdx); - Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[valueGenerators.ckComparators.size()]; + Relations.RelationKind[] lowerBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()]; int opId = opIdCounter++; rngSupplier.doWithSeed(opId, rng -> { - for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckComparators.size()); i++) + for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckColumnCount()); i++) { if (i < nonEqFrom) lowerBoundRelations[i] = Relations.RelationKind.EQ; @@ -552,14 +552,14 @@ public SingleOperationBuilder selectRowSliceByLowerBound(int pdIdx, int lowerBou @Override public SingleOperationBuilder selectRowSliceByUpperBound(int pdIdx, int upperBoundRowIdx, int nonEqFrom, boolean includeBound) { - long pd = valueGenerators.pkGen.descriptorAt(pdIdx); - long upperBoundCd = valueGenerators.ckGen.descriptorAt(upperBoundRowIdx); + long pd = valueGenerators.pkGen().descriptorAt(pdIdx); + long upperBoundCd = valueGenerators.ckGen().descriptorAt(upperBoundRowIdx); - Relations.RelationKind[] upperBoundRelations = new Relations.RelationKind[valueGenerators.ckComparators.size()]; + Relations.RelationKind[] upperBoundRelations = new Relations.RelationKind[valueGenerators.ckColumnCount()]; int opId = opIdCounter++; rngSupplier.doWithSeed(opId, rng -> { - for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckComparators.size()); i++) + for (int i = 0; i < Math.min(nonEqFrom + 1, valueGenerators.ckColumnCount()); i++) { if (i < nonEqFrom) upperBoundRelations[i] = Relations.RelationKind.EQ; diff --git a/test/harry/main/org/apache/cassandra/harry/execution/CQLTesterVisitExecutor.java b/test/harry/main/org/apache/cassandra/harry/execution/CQLTesterVisitExecutor.java index 0a03947bd6b0..0f301d8cb8f0 100644 --- a/test/harry/main/org/apache/cassandra/harry/execution/CQLTesterVisitExecutor.java +++ b/test/harry/main/org/apache/cassandra/harry/execution/CQLTesterVisitExecutor.java @@ -80,7 +80,7 @@ public static ResultSetRow resultSetToRow(SchemaSpec schema, Operations.SelectSt partitionKey[i] = column.type.asServerType().compose(row.getBytes(column.name)); } - pd = schema.valueGenerators.pkGen.deflate(partitionKey); + pd = schema.valueGenerators.pkGen().deflate(partitionKey); } @@ -111,7 +111,7 @@ public static ResultSetRow resultSetToRow(SchemaSpec schema, Operations.SelectSt if (clusteringKey == NIL_KEY) cd = UNSET_DESCR; else - cd = schema.valueGenerators.ckGen.deflate(clusteringKey); + cd = schema.valueGenerators.ckGen().deflate(clusteringKey); } long[] regularColumns = new long[schema.regularColumns.size()]; @@ -123,7 +123,7 @@ public static ResultSetRow resultSetToRow(SchemaSpec schema, Operations.SelectSt if (row.has(column.name)) { Object value = column.type.asServerType().compose(row.getBytes(column.name)); - regularColumns[i] = schema.valueGenerators.regularColumnGens.get(i).deflate(value); + regularColumns[i] = schema.valueGenerators.regularColumnGen(i).deflate(value); } else { @@ -145,7 +145,7 @@ public static ResultSetRow resultSetToRow(SchemaSpec schema, Operations.SelectSt if (row.has(column.name)) { Object value = column.type.asServerType().compose(row.getBytes(column.name)); - staticColumns[i] = schema.valueGenerators.staticColumnGens.get(i).deflate(value); + staticColumns[i] = schema.valueGenerators.staticColumnGen(i).deflate(value); } else { diff --git a/test/harry/main/org/apache/cassandra/harry/execution/InJvmDTestVisitExecutor.java b/test/harry/main/org/apache/cassandra/harry/execution/InJvmDTestVisitExecutor.java index 32d6492e0288..8362a3111bc6 100644 --- a/test/harry/main/org/apache/cassandra/harry/execution/InJvmDTestVisitExecutor.java +++ b/test/harry/main/org/apache/cassandra/harry/execution/InJvmDTestVisitExecutor.java @@ -184,7 +184,7 @@ public static ResultSetRow rowToResultSet(SchemaSpec schema, Operations.SelectSt for (int i = 0; i < schema.partitionKeys.size(); i++) partitionKey[i] = result[selection.indexOf(schema.partitionKeys.get(i))]; - pd = schema.valueGenerators.pkGen.deflate(partitionKey); + pd = schema.valueGenerators.pkGen().deflate(partitionKey); } // Deflate logic for clustering key is a bit more involved, since CK can be nil in case of a single static row. @@ -212,7 +212,7 @@ public static ResultSetRow rowToResultSet(SchemaSpec schema, Operations.SelectSt if (clusteringKey == NIL_KEY) cd = UNSET_DESCR; else - cd = schema.valueGenerators.ckGen.deflate(clusteringKey); + cd = schema.valueGenerators.ckGen().deflate(clusteringKey); } for (int i = 0; i < schema.regularColumns.size(); i++) @@ -224,7 +224,7 @@ public static ResultSetRow rowToResultSet(SchemaSpec schema, Operations.SelectSt if (v == null) regularColumns[i] = NIL_DESCR; else - regularColumns[i] = schema.valueGenerators.regularColumnGens.get(i).deflate(v); + regularColumns[i] = schema.valueGenerators.regularColumnGen(i).deflate(v); } else { @@ -241,7 +241,7 @@ public static ResultSetRow rowToResultSet(SchemaSpec schema, Operations.SelectSt if (v == null) staticColumns[i] = NIL_DESCR; else - staticColumns[i] = schema.valueGenerators.staticColumnGens.get(i).deflate(v); + staticColumns[i] = schema.valueGenerators.staticColumnGen(i).deflate(v); } else { diff --git a/test/harry/main/org/apache/cassandra/harry/execution/ResultSetRow.java b/test/harry/main/org/apache/cassandra/harry/execution/ResultSetRow.java index 144738d45e04..026cf2caaf9c 100644 --- a/test/harry/main/org/apache/cassandra/harry/execution/ResultSetRow.java +++ b/test/harry/main/org/apache/cassandra/harry/execution/ResultSetRow.java @@ -19,11 +19,10 @@ package org.apache.cassandra.harry.execution; import java.util.Arrays; -import java.util.List; import java.util.Objects; +import java.util.function.IntFunction; import org.apache.cassandra.harry.gen.Bijections; -import org.apache.cassandra.harry.MagicConstants; import org.apache.cassandra.harry.gen.ValueGenerators; import org.apache.cassandra.harry.util.StringUtils; @@ -102,33 +101,27 @@ public String toString() ")"; } - private static String toString(long[] descriptors, List> gens) + private static String toString(long[] descriptors, IntFunction> gens) { - int[] idxs = new int[gens.size()]; + String[] idxs = new String[descriptors.length]; for (int i = 0; i < descriptors.length; i++) - idxs[i] = descrToIdx(gens.get(i), descriptors[i]); - return StringUtils.toString(idxs); + idxs[i] = descrToIdx(gens.apply(i), descriptors[i]); + return StringUtils.concat(idxs); } - private static int descrToIdx(Bijections.IndexedBijection gen, long descr) + private static String descrToIdx(Bijections.Bijection gen, long descr) { - if (descr == MagicConstants.UNSET_DESCR) - return MagicConstants.UNSET_IDX; - - if (descr == MagicConstants.NIL_DESCR) - return MagicConstants.NIL_IDX; - - return gen.idxFor(descr); + return gen.toString(descr); } public String toString(ValueGenerators valueGenerators) { return "resultSetRow(" - + valueGenerators.pkGen.idxFor(pd) - + ", " + StringUtils.toString(descrToIdx(valueGenerators.ckGen, cd)) + - (sds == null ? "" : ", statics(" + toString(sds, valueGenerators.staticColumnGens) + ")") + + + valueGenerators.pkGen().toString(pd) + + ", " + descrToIdx(valueGenerators.ckGen(), cd) + + (sds == null ? "" : ", statics(" + toString(sds, valueGenerators::staticColumnGen) + ")") + (slts == null ? "" : ", slts(" + StringUtils.toString(slts) + ")") + - ", values(" + toString(vds, valueGenerators.regularColumnGens) + ")" + + ", values(" + toString(vds, valueGenerators::regularColumnGen) + ")" + ", lts(" + StringUtils.toString(lts) + ")" + ")"; } diff --git a/test/harry/main/org/apache/cassandra/harry/execution/RingAwareInJvmDTestVisitExecutor.java b/test/harry/main/org/apache/cassandra/harry/execution/RingAwareInJvmDTestVisitExecutor.java index 41eb505f3bdb..b9a6e0565d56 100644 --- a/test/harry/main/org/apache/cassandra/harry/execution/RingAwareInJvmDTestVisitExecutor.java +++ b/test/harry/main/org/apache/cassandra/harry/execution/RingAwareInJvmDTestVisitExecutor.java @@ -85,7 +85,7 @@ public List getReplicasFor(long pd) protected long token(long pd) { - return TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(schema.valueGenerators.pkGen.inflate(pd)))); + return TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(schema.valueGenerators.pkGen().inflate(pd)))); } @Override diff --git a/test/harry/main/org/apache/cassandra/harry/gen/Bijections.java b/test/harry/main/org/apache/cassandra/harry/gen/Bijections.java index 454449b71ab4..c9e30ef21700 100644 --- a/test/harry/main/org/apache/cassandra/harry/gen/Bijections.java +++ b/test/harry/main/org/apache/cassandra/harry/gen/Bijections.java @@ -22,6 +22,8 @@ import java.util.Date; import java.util.UUID; +import org.apache.cassandra.harry.MagicConstants; + public class Bijections { public static final Bijection INT8_GENERATOR = new ByteGenerator(); @@ -86,20 +88,11 @@ default Comparator descriptorsComparator() { return Long::compare; } - } - /** - * Indexed bijection allows to decouple descriptor order from value order, which makes data generation simpler. - * - * For regular Harry bijections, this is done at no cost, since values are inflated in a way that preserves - * descriptor order, which means that idx order is consistent with descriptor order and consistent with value order. - * - * An indexed bijection allows order to be established via index, and use descriptor simply as a seed for random values. - */ - public interface IndexedBijection extends Bijection - { - int idxFor(long descriptor); - long descriptorAt(int idx); + default String toString(long pd) + { + return Long.toString(pd); + } } public static long minForSize(int size) diff --git a/test/harry/main/org/apache/cassandra/harry/gen/InvertibleGenerator.java b/test/harry/main/org/apache/cassandra/harry/gen/InvertibleGenerator.java index d71ddff6d7af..8c023e5c8cbd 100644 --- a/test/harry/main/org/apache/cassandra/harry/gen/InvertibleGenerator.java +++ b/test/harry/main/org/apache/cassandra/harry/gen/InvertibleGenerator.java @@ -31,6 +31,7 @@ import org.agrona.collections.IntHashSet; import org.apache.cassandra.harry.ColumnSpec; import org.apache.cassandra.harry.MagicConstants; +import org.apache.cassandra.harry.dsl.HistoryBuilder; import org.apache.cassandra.harry.gen.rng.SeedableEntropySource; import org.apache.cassandra.utils.ArrayUtils; @@ -49,7 +50,7 @@ * TODO (expected): custom invertible generator for bool, u8, u16, u32, etc, for efficiency. * TODO (expected): implement support for tuple/vector/udt, and other multi-cell types. */ -public class InvertibleGenerator implements Bijections.IndexedBijection +public class InvertibleGenerator implements HistoryBuilder.IndexedBijection { public static long MAX_ENTROPY = 1L << 63; diff --git a/test/harry/main/org/apache/cassandra/harry/gen/OperationsGenerators.java b/test/harry/main/org/apache/cassandra/harry/gen/OperationsGenerators.java index 12ba6b170987..33b831bfb6f5 100644 --- a/test/harry/main/org/apache/cassandra/harry/gen/OperationsGenerators.java +++ b/test/harry/main/org/apache/cassandra/harry/gen/OperationsGenerators.java @@ -19,6 +19,7 @@ package org.apache.cassandra.harry.gen; import org.apache.cassandra.harry.SchemaSpec; +import org.apache.cassandra.harry.dsl.HistoryBuilder; import org.apache.cassandra.harry.op.Operations; public class OperationsGenerators @@ -40,7 +41,9 @@ public Long generate(EntropySource rng) // TODO: distributions public static Generator sequentialPd(SchemaSpec schema) { - int population = schema.valueGenerators.pkPopulation(); + // TODO: switch away from Indexed generators here + HistoryBuilder.IndexedValueGenerators valueGenerators = (HistoryBuilder.IndexedValueGenerators) schema.valueGenerators; + int population = valueGenerators.pkPopulation(); return new Generator<>() { @@ -49,14 +52,16 @@ public static Generator sequentialPd(SchemaSpec schema) @Override public Long generate(EntropySource rng) { - return schema.valueGenerators.pkGen.descriptorAt(counter++ % population); + return valueGenerators.pkGen().descriptorAt(counter++ % population); } }; } public static Generator sequentialCd(SchemaSpec schema) { - int population = schema.valueGenerators.ckPopulation(); + // TODO: switch away from Indexed generators here + HistoryBuilder.IndexedValueGenerators valueGenerators = (HistoryBuilder.IndexedValueGenerators) schema.valueGenerators; + int population = valueGenerators.ckPopulation(); return new Generator<>() { @@ -65,7 +70,7 @@ public static Generator sequentialCd(SchemaSpec schema) @Override public Long generate(EntropySource rng) { - return schema.valueGenerators.ckGen.descriptorAt(counter++ % population); + return valueGenerators.ckGen().descriptorAt(counter++ % population); } }; } @@ -80,20 +85,23 @@ public static Generator writeOp(SchemaSpec schema, Generator pdGen, Generator cdGen) { + // TODO: switch away from Indexed generators here + HistoryBuilder.IndexedValueGenerators valueGenerators = (HistoryBuilder.IndexedValueGenerators) schema.valueGenerators; + return (rng) -> { long pd = pdGen.generate(rng); long cd = cdGen.generate(rng); long[] vds = new long[schema.regularColumns.size()]; for (int i = 0; i < schema.regularColumns.size(); i++) { - int idx = rng.nextInt(schema.valueGenerators.regularPopulation(i)); - vds[i] = schema.valueGenerators.regularColumnGens.get(i).descriptorAt(idx); + int idx = rng.nextInt(valueGenerators.regularPopulation(i)); + vds[i] = valueGenerators.regularColumnGen(i).descriptorAt(idx); } long[] sds = new long[schema.staticColumns.size()]; for (int i = 0; i < schema.staticColumns.size(); i++) { - int idx = rng.nextInt(schema.valueGenerators.staticPopulation(i)); - sds[i] = schema.valueGenerators.staticColumnGens.get(i).descriptorAt(idx); + int idx = rng.nextInt(valueGenerators.staticPopulation(i)); + sds[i] = valueGenerators.staticColumnGen(i).descriptorAt(idx); } return lts -> new Operations.WriteOp(lts, pd, cd, vds, sds, Operations.Kind.INSERT); }; diff --git a/test/harry/main/org/apache/cassandra/harry/gen/ValueGenerators.java b/test/harry/main/org/apache/cassandra/harry/gen/ValueGenerators.java index 73c6d519e775..14af3ceacf32 100644 --- a/test/harry/main/org/apache/cassandra/harry/gen/ValueGenerators.java +++ b/test/harry/main/org/apache/cassandra/harry/gen/ValueGenerators.java @@ -18,40 +18,28 @@ package org.apache.cassandra.harry.gen; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.cassandra.harry.ColumnSpec; -import org.apache.cassandra.harry.SchemaSpec; -import org.apache.cassandra.harry.gen.rng.JdkRandomEntropySource; -import org.apache.cassandra.harry.util.IteratorsUtil; - -import static org.apache.cassandra.harry.gen.InvertibleGenerator.fromType; -import static org.apache.cassandra.harry.SchemaSpec.cumulativeEntropy; -import static org.apache.cassandra.harry.SchemaSpec.forKeys; +import org.apache.cassandra.harry.dsl.HistoryBuilder; public class ValueGenerators { - public final Bijections.IndexedBijection pkGen; - public final Bijections.IndexedBijection ckGen; + protected final Bijections.Bijection pkGen; + protected final Bijections.Bijection ckGen; - public final List> regularColumnGens; - public final List> staticColumnGens; + protected final List> regularColumnGens; + protected final List> staticColumnGens; - public final List> pkComparators; - public final List> ckComparators; - public final List> regularComparators; - public final List> staticComparators; + protected final List> pkComparators; + protected final List> ckComparators; + protected final List> regularComparators; + protected final List> staticComparators; - public ValueGenerators(Bijections.IndexedBijection pkGen, - Bijections.IndexedBijection ckGen, - List> regularColumnGens, - List> staticColumnGens, + public ValueGenerators(Bijections.Bijection pkGen, + Bijections.Bijection ckGen, + List> regularColumnGens, + List> staticColumnGens, List> pkComparators, List> ckComparators, @@ -68,41 +56,59 @@ public ValueGenerators(Bijections.IndexedBijection pkGen, this.staticComparators = staticComparators; } - @SuppressWarnings({ "unchecked" }) - public static ValueGenerators fromSchema(SchemaSpec schema, long seed, int populationPerColumn) - { - List> pkComparators = new ArrayList<>(); - List> ckComparators = new ArrayList<>(); - List> regularComparators = new ArrayList<>(); - List> staticComparators = new ArrayList<>(); - - EntropySource rng = new JdkRandomEntropySource(seed); - for (int i = 0; i < schema.partitionKeys.size(); i++) - pkComparators.add((Comparator) schema.partitionKeys.get(i).type.comparator()); - for (int i = 0; i < schema.clusteringKeys.size(); i++) - ckComparators.add((Comparator) schema.clusteringKeys.get(i).type.comparator()); - for (int i = 0; i < schema.regularColumns.size(); i++) - regularComparators.add((Comparator) schema.regularColumns.get(i).type.comparator()); - for (int i = 0; i < schema.staticColumns.size(); i++) - staticComparators.add((Comparator) schema.staticColumns.get(i).type.comparator()); - - Map, InvertibleGenerator> map = new HashMap<>(); - for (ColumnSpec column : IteratorsUtil.concat(schema.regularColumns, schema.staticColumns)) - map.computeIfAbsent(column, (a) -> (InvertibleGenerator) fromType(rng, populationPerColumn, column)); - - // TODO: empty gen - return new ValueGenerators(new InvertibleGenerator<>(rng, cumulativeEntropy(schema.partitionKeys), populationPerColumn, forKeys(schema.partitionKeys), keyComparator(schema.partitionKeys)), - new InvertibleGenerator<>(rng, cumulativeEntropy(schema.clusteringKeys), populationPerColumn, forKeys(schema.clusteringKeys), keyComparator(schema.clusteringKeys)), - schema.regularColumns.stream() - .map(map::get) - .collect(Collectors.toList()), - schema.staticColumns.stream() - .map(map::get) - .collect(Collectors.toList()), - pkComparators, - ckComparators, - regularComparators, - staticComparators); + public Bijections.Bijection pkGen() + { + return pkGen; + } + + public Bijections.Bijection ckGen() + { + return ckGen; + } + + public Bijections.Bijection regularColumnGen(int idx) + { + return regularColumnGens.get(idx); + } + + public Bijections.Bijection staticColumnGen(int idx) + { + return staticColumnGens.get(idx); + } + + public int ckColumnCount() + { + return ckComparators.size(); + } + + public int regularColumnCount() + { + return regularColumnGens.size(); + } + + public int staticColumnCount() + { + return staticColumnGens.size(); + } + + public Comparator pkComparator(int idx) + { + return pkComparators.get(idx); + } + + public Comparator ckComparator(int idx) + { + return ckComparators.get(idx); + } + + public Comparator regularComparator(int idx) + { + return regularComparators.get(idx); + } + + public Comparator staticComparator(int idx) + { + return staticComparators.get(idx); } public int pkPopulation() @@ -124,28 +130,4 @@ public int staticPopulation(int i) { return staticColumnGens.get(i).population(); } - - private static Comparator keyComparator(List> columns) - { - return (o1, o2) -> compareKeys(columns, o1, o2); - } - - public static int compareKeys(List> columns, Object[] v1, Object[] v2) - { - assert v1.length == v2.length : String.format("Values should be of same length: %d != %d\n%s\n%s", - v1.length, v2.length, Arrays.toString(v1), Arrays.toString(v2)); - - for (int i = 0; i < v1.length; i++) - { - int res; - ColumnSpec column = columns.get(i); - if (column.type.isReversed()) - res = column.type.comparator().reversed().compare(v1[i], v2[i]); - else - res = column.type.comparator().compare(v1[i], v2[i]); - if (res != 0) - return res; - } - return 0; - } } \ No newline at end of file diff --git a/test/harry/main/org/apache/cassandra/harry/model/PartitionState.java b/test/harry/main/org/apache/cassandra/harry/model/PartitionState.java index ee5e155ab96b..e6e020515c97 100644 --- a/test/harry/main/org/apache/cassandra/harry/model/PartitionState.java +++ b/test/harry/main/org/apache/cassandra/harry/model/PartitionState.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.NavigableMap; import java.util.TreeMap; +import java.util.function.IntFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,12 +59,12 @@ public class PartitionState implements Iterable public PartitionState(long pd, ValueGenerators valueGenerators) { this.pd = pd; - this.rows = new TreeMap<>(valueGenerators.ckGen.descriptorsComparator()); + this.rows = new TreeMap<>(valueGenerators.ckGen().descriptorsComparator()); this.valueGenerators = valueGenerators; this.staticRow = new RowState(this, STATIC_CLUSTERING, - arr(valueGenerators.staticColumnGens.size(), MagicConstants.NIL_DESCR), - arr(valueGenerators.staticColumnGens.size(), MagicConstants.NO_TIMESTAMP)); + arr(valueGenerators.staticColumnCount(), MagicConstants.NIL_DESCR), + arr(valueGenerators.staticColumnCount(), MagicConstants.NO_TIMESTAMP)); } /** @@ -76,19 +77,20 @@ public NavigableMap rows() public void writeStatic(long[] sds, long lts) { - staticRow = updateRowState(staticRow, valueGenerators.staticColumnGens, STATIC_CLUSTERING, sds, lts, false); + staticRow = updateRowState(staticRow, valueGenerators::staticColumnGen, STATIC_CLUSTERING, sds, lts, false); } public void writeRegular(long cd, long[] vds, long lts, boolean writePrimaryKeyLiveness) { - rows.compute(cd, (cd_, current) -> updateRowState(current, valueGenerators.regularColumnGens, cd, vds, lts, writePrimaryKeyLiveness)); + rows.compute(cd, (cd_, current) -> updateRowState(current, valueGenerators::regularColumnGen, cd, vds, lts, writePrimaryKeyLiveness)); } public void delete(Operations.DeleteRange delete, long lts) { // TODO: inefficient; need to search for lower/higher bounds - rows.entrySet().removeIf(e -> Relations.matchRange(valueGenerators.ckGen, - valueGenerators.ckComparators, + rows.entrySet().removeIf(e -> Relations.matchRange(valueGenerators.ckGen(), + valueGenerators::ckComparator, + valueGenerators.ckColumnCount(), delete.lowerBound(), delete.upperBound(), delete.lowerBoundRelation(), @@ -131,8 +133,9 @@ private void filterInternal(Operations.SelectRow select) private void filterInternal(Operations.SelectRange select) { // TODO: inefficient; need to search for lower/higher bounds - rows.entrySet().removeIf(e -> !Relations.matchRange(valueGenerators.ckGen, - valueGenerators.ckComparators, + rows.entrySet().removeIf(e -> !Relations.matchRange(valueGenerators.ckGen(), + valueGenerators::ckComparator, + valueGenerators.ckColumnCount(), select.lowerBound(), select.upperBound(), select.lowerBoundRelation(), @@ -147,31 +150,31 @@ private void filterInternal(Operations.SelectCustom select) Map cache = new HashMap<>(); for (Relations.Relation relation : select.ckRelations()) { - Object[] query = cache.computeIfAbsent(relation.descriptor, valueGenerators.ckGen::inflate); - Object[] match = cache.computeIfAbsent(e.getValue().cd, valueGenerators.ckGen::inflate); - if (!relation.kind.match(valueGenerators.ckComparators.get(relation.column), match[relation.column], query[relation.column])) + Object[] query = cache.computeIfAbsent(relation.descriptor, valueGenerators.ckGen()::inflate); + Object[] match = cache.computeIfAbsent(e.getValue().cd, valueGenerators.ckGen()::inflate); + if (!relation.kind.match(valueGenerators.ckComparator(relation.column), match[relation.column], query[relation.column])) return true; // true means "no match", so remove from resultset } for (Relations.Relation relation : select.regularRelations()) { - Object query = valueGenerators.regularColumnGens.get(relation.column).inflate(relation.descriptor); + Object query = valueGenerators.regularColumnGen(relation.column).inflate(relation.descriptor); long descriptor = e.getValue().vds[relation.column]; if (MagicConstants.MAGIC_DESCRIPTOR_VALS.contains(descriptor)) // TODO: do we allow UNSET queries? return true; - Object match = valueGenerators.regularColumnGens.get(relation.column).inflate(e.getValue().vds[relation.column]); - if (!relation.kind.match(valueGenerators.regularComparators.get(relation.column), match, query)) + Object match = valueGenerators.regularColumnGen(relation.column).inflate(e.getValue().vds[relation.column]); + if (!relation.kind.match(valueGenerators.regularComparator(relation.column), match, query)) return true; } for (Relations.Relation relation : select.staticRelations()) { - Object query = valueGenerators.staticColumnGens.get(relation.column).inflate(relation.descriptor); + Object query = valueGenerators.staticColumnGen(relation.column).inflate(relation.descriptor); long descriptor = e.getValue().partitionState.staticRow.vds[relation.column]; if (MagicConstants.MAGIC_DESCRIPTOR_VALS.contains(descriptor)) // TODO: do we allow UNSET queries? return true; - Object match = valueGenerators.staticColumnGens.get(relation.column).inflate(e.getValue().partitionState.staticRow.vds[relation.column]); - if (!relation.kind.match(valueGenerators.staticComparators.get(relation.column), match, query)) + Object match = valueGenerators.staticColumnGen(relation.column).inflate(e.getValue().partitionState.staticRow.vds[relation.column]); + if (!relation.kind.match(valueGenerators.staticComparator(relation.column), match, query)) return true; } @@ -197,7 +200,7 @@ public boolean isEmpty() /** * Method used to update row state of both static and regular rows. */ - private RowState updateRowState(RowState currentState, List> columns, long cd, long[] vds, long lts, boolean writePrimaryKeyLiveness) + private RowState updateRowState(RowState currentState, IntFunction> columns, long cd, long[] vds, long lts, boolean writePrimaryKeyLiveness) { if (currentState == null) { @@ -232,7 +235,7 @@ private RowState updateRowState(RowState currentState, List column = columns.get(i); + Bijections.Bijection column = columns.apply(i); if (column.compare(vds[i], currentState.vds[i]) > 0) currentState.vds[i] = vds[i]; } @@ -366,23 +369,17 @@ public RowState clone() return rowState; } - private static String toString(long[] descriptors, List> gens) + private static String toString(long[] descriptors, IntFunction> gens) { - int[] idxs = new int[gens.size()]; + String[] idxs = new String[descriptors.length]; for (int i = 0; i < descriptors.length; i++) - idxs[i] = descrToIdxForToString(gens.get(i), descriptors[i]); - return StringUtils.toString(idxs); + idxs[i] = descrToIdxForToString(gens.apply(i), descriptors[i]); + return StringUtils.concat(idxs); } - public static int descrToIdxForToString(Bijections.IndexedBijection gen, long descr) + public static String descrToIdxForToString(Bijections.Bijection gen, long descr) { - if (descr == MagicConstants.UNSET_DESCR) - return MagicConstants.UNSET_IDX; - - if (descr == MagicConstants.NIL_DESCR) - return MagicConstants.NIL_IDX; - - return gen.idxFor(descr); + return gen.toString(descr); } public String toString(ValueGenerators valueGenerators) @@ -390,17 +387,17 @@ public String toString(ValueGenerators valueGenerators) if (cd == STATIC_CLUSTERING) { return " rowStateRow(" - + valueGenerators.pkGen.idxFor(partitionState.pd) + + + valueGenerators.pkGen().toString(partitionState.pd) + ", STATIC" + - ", statics(" + toString(partitionState.staticRow.vds, valueGenerators.staticColumnGens) + ")" + + ", statics(" + toString(partitionState.staticRow.vds, valueGenerators::staticColumnGen) + ")" + ", lts(" + StringUtils.toString(partitionState.staticRow.lts) + ")"; } else { return " rowStateRow(" - + valueGenerators.pkGen.idxFor(partitionState.pd) + - ", " + descrToIdxForToString(valueGenerators.ckGen, cd) + - ", vds(" + toString(vds, valueGenerators.regularColumnGens) + ")" + + + valueGenerators.pkGen().toString(partitionState.pd) + + ", " + descrToIdxForToString(valueGenerators.ckGen(), cd) + + ", vds(" + toString(vds, valueGenerators::regularColumnGen) + ")" + ", lts(" + StringUtils.toString(lts) + ")"; } } diff --git a/test/harry/main/org/apache/cassandra/harry/model/PartitionStateBuilder.java b/test/harry/main/org/apache/cassandra/harry/model/PartitionStateBuilder.java index 110e440b8e65..d67a9ed21517 100644 --- a/test/harry/main/org/apache/cassandra/harry/model/PartitionStateBuilder.java +++ b/test/harry/main/org/apache/cassandra/harry/model/PartitionStateBuilder.java @@ -118,7 +118,7 @@ protected void endLts(long lts) if (hadTrackingRowWrite) { - long[] statics = new long[valueGenerators.staticColumnGens.size()]; + long[] statics = new long[valueGenerators.staticColumnCount()]; Arrays.fill(statics, MagicConstants.UNSET_DESCR); partitionState.writeStatic(statics, lts); } diff --git a/test/harry/main/org/apache/cassandra/harry/test/HistoryBuilderTest.java b/test/harry/main/org/apache/cassandra/harry/test/HistoryBuilderTest.java index 24d9e75689f0..ec864b10e46d 100644 --- a/test/harry/main/org/apache/cassandra/harry/test/HistoryBuilderTest.java +++ b/test/harry/main/org/apache/cassandra/harry/test/HistoryBuilderTest.java @@ -32,7 +32,6 @@ import org.apache.cassandra.harry.checker.ModelChecker; import org.apache.cassandra.harry.dsl.HistoryBuilder; import org.apache.cassandra.harry.dsl.HistoryBuilderHelper; -import org.apache.cassandra.harry.dsl.SingleOperationBuilder; import org.apache.cassandra.harry.execution.CQLTesterVisitExecutor; import org.apache.cassandra.harry.execution.CQLVisitExecutor; import org.apache.cassandra.harry.execution.DataTracker; @@ -104,7 +103,7 @@ public void orderByTest() history.insert(1); history.custom((lts, opId) -> new Operations.SelectPartition(lts, - schema.valueGenerators.pkGen.descriptorAt(1), + history.valueGenerators().pkGen().descriptorAt(1), Operations.ClusteringOrderBy.DESC)); replay(schema, history); @@ -202,7 +201,7 @@ public void testSimpleFuzz() Generator partitionPicker = Generators.pick(0, maxPartitions); Generator rowPicker = Generators.int32(0, maxPartitionSize); - ModelChecker modelChecker = new ModelChecker<>(); + ModelChecker modelChecker = new ModelChecker<>(); HistoryBuilder historyBuilder = new HistoryBuilder(schema.valueGenerators); modelChecker.init(historyBuilder) .step((history, rng_) -> { @@ -271,7 +270,7 @@ public void fuzzFiltering() Generator pkGen = Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), maxPartitionSize)); Generator ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), maxPartitionSize)); - ModelChecker modelChecker = new ModelChecker<>(); + ModelChecker modelChecker = new ModelChecker<>(); HistoryBuilder historyBuilder = new HistoryBuilder(schema.valueGenerators); modelChecker.init(historyBuilder) diff --git a/test/harry/main/org/apache/cassandra/harry/test/SimpleBijectionTest.java b/test/harry/main/org/apache/cassandra/harry/test/SimpleBijectionTest.java index 57d8f52eb636..6e33573f6fb1 100644 --- a/test/harry/main/org/apache/cassandra/harry/test/SimpleBijectionTest.java +++ b/test/harry/main/org/apache/cassandra/harry/test/SimpleBijectionTest.java @@ -26,6 +26,7 @@ import accord.utils.Invariants; import org.apache.cassandra.harry.ColumnSpec; +import org.apache.cassandra.harry.dsl.HistoryBuilder; import org.apache.cassandra.harry.gen.InvertibleGenerator; import org.apache.cassandra.harry.SchemaSpec; import org.apache.cassandra.harry.gen.ValueGenerators; @@ -76,14 +77,14 @@ public void testArrayOrder() MAX_ENTROPY, 100, SchemaSpec.forKeys(columns), - (Object[] a, Object[] b) -> ValueGenerators.compareKeys(columns, a, b)); + (Object[] a, Object[] b) -> HistoryBuilder.compareKeys(columns, a, b)); Object[] previous = null; for (int i = 0; i < 100; i++) { long descr = generator.descriptorAt(i); Object[] next = generator.inflate(descr); if (previous != null) - Assert.assertTrue(ValueGenerators.compareKeys(columns, next, previous) > 0); + Assert.assertTrue( HistoryBuilder.compareKeys(columns, next, previous) > 0); Assert.assertEquals(descr, generator.deflate(next)); previous = next; } diff --git a/test/harry/main/org/apache/cassandra/harry/util/StringUtils.java b/test/harry/main/org/apache/cassandra/harry/util/StringUtils.java index f6571c44784a..707b50565502 100644 --- a/test/harry/main/org/apache/cassandra/harry/util/StringUtils.java +++ b/test/harry/main/org/apache/cassandra/harry/util/StringUtils.java @@ -22,6 +22,17 @@ public class StringUtils { + public static String concat(String[] arr) + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < arr.length; i++) + { + sb.append(arr[i]); + if (i < arr.length - 1) sb.append(','); + } + return sb.toString(); + } + public static String toString(long[] arr) { if (arr.length == 0) diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/HarryValidatingQuery.java b/test/simulator/test/org/apache/cassandra/simulator/test/HarryValidatingQuery.java index 2ca9d9b915a6..2a011c944c74 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/HarryValidatingQuery.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/HarryValidatingQuery.java @@ -101,7 +101,7 @@ public void run() protected long token(long pd) { - return TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(simulation.schema.valueGenerators.pkGen.inflate(pd)))); + return TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(simulation.schema.valueGenerators.pkGen().inflate(pd)))); } protected Object[][] executeNodeLocal(String statement, TokenPlacementModel.Node node, Object... bindings)