diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElements.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElements.java index 7494a61c1..be9f1b5cb 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElements.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElements.java @@ -3,12 +3,17 @@ import com.netflix.hollow.core.memory.MemoryMode; import com.netflix.hollow.core.memory.encoding.GapEncodedVariableLengthIntegerReader; import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler; +import com.netflix.hollow.core.read.HollowBlobInput; +import com.netflix.hollow.core.read.engine.map.HollowMapTypeDataElements; +import java.io.IOException; public abstract class AbstractHollowTypeDataElements { public int maxOrdinal; + public GapEncodedVariableLengthIntegerReader encodedAdditions; public GapEncodedVariableLengthIntegerReader encodedRemovals; + public final ArraySegmentRecycler memoryRecycler; public final MemoryMode memoryMode; @@ -16,4 +21,6 @@ public AbstractHollowTypeDataElements(MemoryMode memoryMode, ArraySegmentRecycle this.memoryMode = memoryMode; this.memoryRecycler = memoryRecycler; } + + public abstract void destroy(); } diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsJoiner.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsJoiner.java index 4a108656f..47c5a0285 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsJoiner.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsJoiner.java @@ -20,17 +20,18 @@ public AbstractHollowTypeDataElementsJoiner(T[] from) { for (AbstractHollowTypeDataElements elements : from) { if (elements.encodedAdditions != null) { - throw new IllegalStateException("Encountered encodedAdditions in data elements splitter- this is not expected " + + throw new IllegalStateException("Encountered encodedAdditions in data elements joiner- this is not expected " + "since encodedAdditions only exist on delta data elements and they dont carry over to target data elements, " + "delta data elements are never split/joined"); } } + + // SNAP: TODO: should support a "too large to join" when joined ordinals will overflow } public T join() { - init(); - + initToElements(); to.maxOrdinal = -1; populateStats(); @@ -43,13 +44,22 @@ public T join() { } to.encodedRemovals = GapEncodedVariableLengthIntegerReader.join(fromRemovals); - return (T) to; + return to; } - public abstract void init(); + /** + * Initialize the target data elements. + */ + public abstract void initToElements(); + /** + * Populate the stats of the target data elements. + */ public abstract void populateStats(); + /** + * Copy records from the source data elements to the target data elements. + */ public abstract void copyRecords(); diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsSplitter.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsSplitter.java index 8d763df87..2065bae7d 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsSplitter.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsSplitter.java @@ -2,6 +2,12 @@ import com.netflix.hollow.core.memory.encoding.GapEncodedVariableLengthIntegerReader; +/** + * Join multiple {@code HollowListTypeDataElements}s into 1 {@code HollowListTypeDataElements}. + * Ordinals are remapped and corresponding data is copied over. + * The original data elements are not destroyed. + * The no. of passed data elements must be a power of 2. + */ public abstract class AbstractHollowTypeDataElementsSplitter { public final int numSplits; public final int toMask; @@ -10,7 +16,7 @@ public abstract class AbstractHollowTypeDataElementsSplitter> fromOrdinalShift; @@ -65,7 +67,7 @@ public void copyRecords() { to.copyElementsFrom(elementCounter, source, startElement, endElement); elementCounter += numElements; } - to.listPointerData.setElementValue((long) to.bitsPerListPointer * ordinal, to.bitsPerListPointer, elementCounter); + to.listPointerData.setElementValue((long)to.bitsPerListPointer * ordinal, to.bitsPerListPointer, elementCounter); } } } diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitter.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitter.java index 6562ac574..534005b6a 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitter.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitter.java @@ -16,7 +16,7 @@ public HollowListTypeDataElementsSplitter(HollowListTypeDataElements from, int n } @Override - public void init() { + public void initToElements() { this.to = new HollowListTypeDataElements[numSplits]; for(int i=0;i> fromOrdinalShift; @@ -82,10 +83,10 @@ public void copyRecords() { to.copyBucketsFrom(bucketCounter, source, startBucket, endBucket); bucketCounter += numBuckets; - mapSize = source.mapPointerAndSizeData.getElementValue((long) (fromOrdinal * source.bitsPerFixedLengthMapPortion) + source.bitsPerMapPointer, source.bitsPerMapSizeValue); + mapSize = source.mapPointerAndSizeData.getElementValue((long)(fromOrdinal * source.bitsPerFixedLengthMapPortion) + source.bitsPerMapPointer, source.bitsPerMapSizeValue); } - to.mapPointerAndSizeData.setElementValue( (long) ordinal * to.bitsPerFixedLengthMapPortion, to.bitsPerMapPointer, bucketCounter); - to.mapPointerAndSizeData.setElementValue((long) (ordinal * to.bitsPerFixedLengthMapPortion) + to.bitsPerMapPointer, to.bitsPerMapSizeValue, mapSize); + to.mapPointerAndSizeData.setElementValue( (long)ordinal * to.bitsPerFixedLengthMapPortion, to.bitsPerMapPointer, bucketCounter); + to.mapPointerAndSizeData.setElementValue((long)(ordinal * to.bitsPerFixedLengthMapPortion) + to.bitsPerMapPointer, to.bitsPerMapSizeValue, mapSize); } } } diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitter.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitter.java index 65693340a..45542418c 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitter.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitter.java @@ -17,7 +17,7 @@ public HollowMapTypeDataElementsSplitter(HollowMapTypeDataElements from, int num } @Override - public void init() { + public void initToElements() { this.to = new HollowMapTypeDataElements[numSplits]; for(int i=0;i> toOrdinalShift; @@ -47,20 +46,17 @@ public void populateStats() { for(int toIndex=0;toIndex> toOrdinalShift; @@ -82,9 +83,9 @@ public void copyRecords() { target.copyBucketsFrom(bucketCounter[toIndex], from, startBucket, endBucket); bucketCounter[toIndex] += numBuckets; - target.mapPointerAndSizeData.setElementValue((long) toOrdinal * target.bitsPerFixedLengthMapPortion, target.bitsPerMapPointer, bucketCounter[toIndex]); - long mapSize = from.mapPointerAndSizeData.getElementValue((long) (ordinal * from.bitsPerFixedLengthMapPortion) + from.bitsPerMapPointer, from.bitsPerMapSizeValue); - target.mapPointerAndSizeData.setElementValue((long) (toOrdinal * target.bitsPerFixedLengthMapPortion) + target.bitsPerMapPointer, target.bitsPerMapSizeValue, mapSize); + target.mapPointerAndSizeData.setElementValue((long)toOrdinal * target.bitsPerFixedLengthMapPortion, target.bitsPerMapPointer, bucketCounter[toIndex]); + long mapSize = from.mapPointerAndSizeData.getElementValue((long)(ordinal * from.bitsPerFixedLengthMapPortion) + from.bitsPerMapPointer, from.bitsPerMapSizeValue); + target.mapPointerAndSizeData.setElementValue((long)(toOrdinal * target.bitsPerFixedLengthMapPortion) + target.bitsPerMapPointer, target.bitsPerMapSizeValue, mapSize); } } } diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadState.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadState.java index 155d1e5f2..6c35cd591 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadState.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeReadState.java @@ -85,9 +85,10 @@ public HollowMapTypeReadState(HollowReadStateEngine stateEngine, MemoryMode memo } - public HollowMapTypeReadState(MemoryMode memoryMode, HollowMapSchema schema, int numShards, HollowMapTypeReadStateShard[] shards) { + HollowMapTypeReadState(MemoryMode memoryMode, HollowMapSchema schema, HollowMapTypeReadStateShard[] shards) { super(null, memoryMode, schema); this.sampler = new HollowMapSampler(schema.getName(), DisabledSamplingDirector.INSTANCE); + int numShards = shards.length; this.shardNumberMask = numShards - 1; this.shardOrdinalShift = 31 - Integer.numberOfLeadingZeros(numShards); diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java index 97c83c15e..7a3411568 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java @@ -38,7 +38,7 @@ */ public class HollowObjectTypeDataElements extends AbstractHollowTypeDataElements { - public final HollowObjectSchema schema; + final HollowObjectSchema schema; FixedLengthData fixedLengthData; final VariableLengthData varLengthData[]; @@ -191,10 +191,11 @@ static void discardFromInput(HollowBlobInput in, HollowObjectSchema schema, int } } - void applyDelta(HollowObjectTypeDataElements fromData, HollowObjectTypeDataElements deltaData) { + public void applyDelta(HollowObjectTypeDataElements fromData, HollowObjectTypeDataElements deltaData) { new HollowObjectDeltaApplicator(fromData, deltaData, this).applyDelta(); } + @Override public void destroy() { FixedLengthDataFactory.destroy(fixedLengthData, memoryRecycler); for(int i=0;i> fromOrdinalShift; @@ -86,7 +87,7 @@ public void copyRecords() { if (fromOrdinal <= from[fromIndex].maxOrdinal) { copyRecord(to, ordinal, from[fromIndex], fromOrdinal, currentWriteVarLengthDataPointers); } else { - // lopsided shards could result for consumers that skip type shards with no additions + // handle lopsided shards that could result from e.g. consumers skipping type shards with no additions writeNullRecord(to, ordinal, currentWriteVarLengthDataPointers); } } diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitter.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitter.java index d31eab305..9f45e2034 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitter.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitter.java @@ -23,7 +23,7 @@ public class HollowObjectTypeDataElementsSplitter extends AbstractHollowTypeData } @Override - public void init() { + public void initToElements() { this.to = new HollowObjectTypeDataElements[numSplits]; for(int i=0;i> fromOrdinalShift; @@ -79,10 +80,10 @@ public void copyRecords() { to.copyBucketsFrom(bucketCounter, source, startBucket, endBucket); bucketCounter += numBuckets; - setSize = source.setPointerAndSizeData.getElementValue((long) (fromOrdinal * source.bitsPerFixedLengthSetPortion) + source.bitsPerSetPointer, source.bitsPerSetSizeValue); + setSize = source.setPointerAndSizeData.getElementValue((long)(fromOrdinal * source.bitsPerFixedLengthSetPortion) + source.bitsPerSetPointer, source.bitsPerSetSizeValue); } - to.setPointerAndSizeData.setElementValue((long) ordinal * to.bitsPerFixedLengthSetPortion, to.bitsPerSetPointer, bucketCounter); - to.setPointerAndSizeData.setElementValue((long) (ordinal * to.bitsPerFixedLengthSetPortion) + to.bitsPerSetPointer, to.bitsPerSetSizeValue, setSize); + to.setPointerAndSizeData.setElementValue((long)ordinal * to.bitsPerFixedLengthSetPortion, to.bitsPerSetPointer, bucketCounter); + to.setPointerAndSizeData.setElementValue((long)(ordinal * to.bitsPerFixedLengthSetPortion) + to.bitsPerSetPointer, to.bitsPerSetSizeValue, setSize); } } } diff --git a/hollow/src/main/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeDataElementsSplitter.java b/hollow/src/main/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeDataElementsSplitter.java index 23d1c30b0..8769afc6f 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeDataElementsSplitter.java +++ b/hollow/src/main/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeDataElementsSplitter.java @@ -16,7 +16,7 @@ public HollowSetTypeDataElementsSplitter(HollowSetTypeDataElements from, int num } @Override - public void init() { + public void initToElements() { this.to = new HollowSetTypeDataElements[numSplits]; for(int i=0;i> toOrdinalShift; @@ -46,18 +45,15 @@ public void populateStats() { for(int toIndex=0;toIndex