Skip to content

Commit

Permalink
Refactoring start/end element/bucket across all collection types
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sunjeet
Sunjeet committed Aug 29, 2024
1 parent 067bb9a commit 5e388de
Show file tree
Hide file tree
Showing 18 changed files with 165 additions and 182 deletions.
4 changes: 2 additions & 2 deletions .../java/com/netflix/hollow/core/read/engine/list/HollowListDeltaHistoricalStateCreator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ private void copyRecord(int ordinal) {
int shardOrdinal = ordinal >> shardOrdinalShift;

long bitsPerElement = stateEngineDataElements[shard].bitsPerElement;
long fromStartElement = shardOrdinal == 0 ? 0 : stateEngineDataElements[shard].listPointerData.getElementValue((long)(shardOrdinal - 1) * stateEngineDataElements[shard].bitsPerListPointer, stateEngineDataElements[shard].bitsPerListPointer);
long fromEndElement = stateEngineDataElements[shard].listPointerData.getElementValue((long)shardOrdinal * stateEngineDataElements[shard].bitsPerListPointer, stateEngineDataElements[shard].bitsPerListPointer);
long fromStartElement = stateEngineDataElements[shard].getStartElement(shardOrdinal);
long fromEndElement = stateEngineDataElements[shard].getEndElement(shardOrdinal);
long size = fromEndElement - fromStartElement;

historicalDataElements.elementData.copyBits(stateEngineDataElements[shard].elementData, fromStartElement * bitsPerElement, nextStartElement * bitsPerElement, size * bitsPerElement);
Expand Down
7 changes: 7 additions & 0 deletions ...ow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElements.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,4 +106,11 @@ public void destroy() {
FixedLengthDataFactory.destroy(elementData, memoryRecycler);
}

public long getStartElement(int ordinal) {
return ordinal == 0 ? 0 : listPointerData.getElementValue(((long) (ordinal-1) * bitsPerListPointer), bitsPerListPointer);
}

public long getEndElement(int ordinal) {
return listPointerData.getElementValue((long)ordinal * bitsPerListPointer, bitsPerListPointer);
}
}
43 changes: 17 additions & 26 deletions .../main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsJoiner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,8 @@ public void populateStats() {
int fromIndex = ordinal & fromMask;
int fromOrdinal = ordinal >> fromOrdinalShift;

long startElement;
long endElement;
if (fromOrdinal == 0) {
startElement = 0;
endElement = from[fromIndex].listPointerData.getElementValue(0, from[fromIndex].bitsPerListPointer);
} else {
long endFixedLengthOffset = (long) fromOrdinal * from[fromIndex].bitsPerListPointer;
long startFixedLengthOffset = endFixedLengthOffset - from[fromIndex].bitsPerListPointer;
startElement = from[fromIndex].listPointerData.getElementValue(startFixedLengthOffset, from[fromIndex].bitsPerListPointer);
endElement = from[fromIndex].listPointerData.getElementValue(endFixedLengthOffset, from[fromIndex].bitsPerListPointer);
}
long startElement = from[fromIndex].getStartElement(fromOrdinal);
long endElement = from[fromIndex].getEndElement(fromOrdinal);
long numElements = endElement - startElement;
totalOfListSizes += numElements;

Expand All @@ -65,23 +56,23 @@ public void copyRecords() {
int fromIndex = ordinal & fromMask;
int fromOrdinal = ordinal >> fromOrdinalShift;

long startElement;
long endElement;
if (fromOrdinal <= from[fromIndex].maxOrdinal) {
if (fromOrdinal == 0) {
startElement = 0;
endElement = from[fromIndex].listPointerData.getElementValue(0, from[fromIndex].bitsPerListPointer);
HollowListTypeDataElements source = from[fromIndex];
long startElement = source.getStartElement(fromOrdinal);
long endElement = source.getEndElement(fromOrdinal);

if (source.bitsPerElement == to.bitsPerElement) {
// fastpath can bulk copy elements. emptyBucketValue is same since bitsPerElement is same
long numElements = endElement - startElement;
int bitsPerElement = source.bitsPerElement;
to.elementData.copyBits(source.elementData, startElement * bitsPerElement, elementCounter * bitsPerElement, numElements * bitsPerElement);
elementCounter += numElements;
} else {
long endFixedLengthOffset = (long)fromOrdinal * from[fromIndex].bitsPerListPointer;
long startFixedLengthOffset = endFixedLengthOffset - from[fromIndex].bitsPerListPointer;
startElement = from[fromIndex].listPointerData.getElementValue(startFixedLengthOffset, from[fromIndex].bitsPerListPointer);
endElement = from[fromIndex].listPointerData.getElementValue(endFixedLengthOffset, from[fromIndex].bitsPerListPointer);
}
// SNAP: TODO: fastpath
for (long element=startElement;element<endElement;element++) {
int elementOrdinal = (int)from[fromIndex].elementData.getElementValue(element * from[fromIndex].bitsPerElement, from[fromIndex].bitsPerElement);
to.elementData.setElementValue(elementCounter * to.bitsPerElement, to.bitsPerElement, elementOrdinal);
elementCounter++;
for (long element = startElement; element < endElement; element++) {
int elementOrdinal = (int) source.elementData.getElementValue(element * source.bitsPerElement, source.bitsPerElement);
to.elementData.setElementValue(elementCounter * to.bitsPerElement, to.bitsPerElement, elementOrdinal);
elementCounter++;
}
}
} // else: lopsided shards could result for consumers that skip type shards with no additions, that gets handled
// by not writing anything to elementData, and writing the cached value of elementCounter to listPointerData
Expand Down
44 changes: 17 additions & 27 deletions ...ain/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,8 @@ public void populateStats() {
int toOrdinal = ordinal >> toOrdinalShift;
to[toIndex].maxOrdinal = toOrdinal;

long startElement;
long endElement;
if (ordinal == 0) {
startElement = 0;
endElement = from.listPointerData.getElementValue(0, from.bitsPerListPointer);
} else {
long endFixedLengthOffset = (long) ordinal * from.bitsPerListPointer;
long startFixedLengthOffset = endFixedLengthOffset - from.bitsPerListPointer;
startElement = from.listPointerData.getElementValue(startFixedLengthOffset, from.bitsPerListPointer);
endElement = from.listPointerData.getElementValue(endFixedLengthOffset, from.bitsPerListPointer);
}

long startElement = from.getStartElement(ordinal);
long endElement = from.getEndElement(ordinal);
long numElements = endElement - startElement;
totalOfListSizes[toIndex] += numElements;
}
Expand Down Expand Up @@ -77,25 +67,25 @@ public void copyRecords() {
int toIndex = ordinal & toMask;
int toOrdinal = ordinal >> toOrdinalShift;

long startElement;
long endElement;
long startElement = from.getStartElement(ordinal);
long endElement = from.getEndElement(ordinal);
HollowListTypeDataElements target = to[toIndex];

if (ordinal == 0) {
startElement = 0;
endElement = from.listPointerData.getElementValue(0, from.bitsPerListPointer);
if (from.bitsPerElement == target.bitsPerElement) {
// fastpath can bulk copy elements. emptyBucketValue is same since bitsPerElement is same
long numElements = endElement - startElement;
int bitsPerElement = from.bitsPerElement;
target.elementData.copyBits(from.elementData, startElement * bitsPerElement, elementCounter[toIndex] * bitsPerElement, numElements * bitsPerElement);
elementCounter[toIndex] += numElements;
} else {
long endFixedLengthOffset = (long)ordinal * from.bitsPerListPointer;
long startFixedLengthOffset = endFixedLengthOffset - from.bitsPerListPointer;
startElement = from.listPointerData.getElementValue(startFixedLengthOffset, from.bitsPerListPointer);
endElement = from.listPointerData.getElementValue(endFixedLengthOffset, from.bitsPerListPointer);
for (long element=startElement;element<endElement;element++) {
int elementOrdinal = (int)from.elementData.getElementValue(element * from.bitsPerElement, from.bitsPerElement);
target.elementData.setElementValue(elementCounter[toIndex] * target.bitsPerElement, target.bitsPerElement, elementOrdinal);
elementCounter[toIndex]++;
}
throw new RuntimeException("Unexpected for List type during split, expected during join"); // SNAP: TODO: remove, or keep for now so that we can test the slow path and in future when we make size optimizations then consumers will be backwards compatible
}

HollowListTypeDataElements target = to[toIndex];
for (long element=startElement;element<endElement;element++) {
int elementOrdinal = (int)from.elementData.getElementValue(element * from.bitsPerElement, from.bitsPerElement);
target.elementData.setElementValue(elementCounter[toIndex] * target.bitsPerElement, target.bitsPerElement, elementOrdinal);
elementCounter[toIndex]++;
}
target.listPointerData.setElementValue((long) target.bitsPerListPointer * toOrdinal, target.bitsPerListPointer, elementCounter[toIndex]);
}
}
Expand Down
27 changes: 4 additions & 23 deletions .../src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadStateShard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,8 @@ public int getElementOrdinal(int ordinal, int listIndex) {

do {
currentData = this.currentDataVolatile;

if (ordinal == 0) {
startElement = 0;
endElement = currentData.listPointerData.getElementValue(0, currentData.bitsPerListPointer);
} else {
long endFixedLengthOffset = (long)ordinal * currentData.bitsPerListPointer;
long startFixedLengthOffset = endFixedLengthOffset - currentData.bitsPerListPointer;
startElement = currentData.listPointerData.getElementValue(startFixedLengthOffset, currentData.bitsPerListPointer);
endElement = currentData.listPointerData.getElementValue(endFixedLengthOffset, currentData.bitsPerListPointer);
}
startElement = currentData.getStartElement(ordinal);
endElement = currentData.getEndElement(ordinal);
} while(readWasUnsafe(currentData));

long elementIndex = startElement + listIndex;
Expand All @@ -65,19 +57,8 @@ public int size(int ordinal) {

do {
currentData = this.currentDataVolatile;

long startElement;
long endElement;
if (ordinal == 0) {
startElement = 0;
endElement = currentData.listPointerData.getElementValue(0, currentData.bitsPerListPointer);
} else {
long endFixedLengthOffset = (long)ordinal * currentData.bitsPerListPointer;
long startFixedLengthOffset = endFixedLengthOffset - currentData.bitsPerListPointer;
startElement = currentData.listPointerData.getElementValue(startFixedLengthOffset, currentData.bitsPerListPointer);
endElement = currentData.listPointerData.getElementValue(endFixedLengthOffset, currentData.bitsPerListPointer);
}

long startElement = currentData.getStartElement(ordinal);
long endElement = currentData.getEndElement(ordinal);
size = (int)(endElement - startElement);
} while(readWasUnsafe(currentData));

Expand Down
4 changes: 2 additions & 2 deletions ...in/java/com/netflix/hollow/core/read/engine/map/HollowMapDeltaHistoricalStateCreator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,8 @@ private void copyRecord(int ordinal) {
long bitsPerBucket = historicalDataElements.bitsPerMapEntry;
long size = typeState.size(ordinal);

long fromStartBucket = shardOrdinal == 0 ? 0 : stateEngineDataElements[shard].mapPointerAndSizeData.getElementValue((long)(shardOrdinal - 1) * stateEngineDataElements[shard].bitsPerFixedLengthMapPortion, stateEngineDataElements[shard].bitsPerMapPointer);
long fromEndBucket = stateEngineDataElements[shard].mapPointerAndSizeData.getElementValue((long)shardOrdinal * stateEngineDataElements[shard].bitsPerFixedLengthMapPortion, stateEngineDataElements[shard].bitsPerMapPointer);
long fromStartBucket = stateEngineDataElements[shard].getStartBucket(shardOrdinal);
long fromEndBucket = stateEngineDataElements[shard].getEndBucket(shardOrdinal);
long numBuckets = fromEndBucket - fromStartBucket;

historicalDataElements.mapPointerAndSizeData.setElementValue((long)nextOrdinal * historicalDataElements.bitsPerFixedLengthMapPortion, historicalDataElements.bitsPerMapPointer, nextStartBucket + numBuckets);
Expand Down
16 changes: 16 additions & 0 deletions hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElements.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,4 +118,20 @@ public void destroy() {
FixedLengthDataFactory.destroy(entryData, memoryRecycler);
}

public long getStartBucket(int ordinal) {
return ordinal == 0 ? 0 : mapPointerAndSizeData.getElementValue((long)(ordinal - 1) * bitsPerFixedLengthMapPortion, bitsPerMapPointer);
}

public long getEndBucket(int ordinal) {
return mapPointerAndSizeData.getElementValue((long)ordinal * bitsPerFixedLengthMapPortion, bitsPerMapPointer);
}

int getBucketKeyByAbsoluteIndex(long absoluteBucketIndex) {
return (int)entryData.getElementValue(absoluteBucketIndex * bitsPerMapEntry, bitsPerKeyElement);
}

int getBucketValueByAbsoluteIndex(long absoluteBucketIndex) {
return (int)entryData.getElementValue((absoluteBucketIndex * bitsPerMapEntry) + bitsPerKeyElement, bitsPerValueElement);
}

}
10 changes: 4 additions & 6 deletions ...rc/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsJoiner.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package com.netflix.hollow.core.read.engine.map;

import static com.netflix.hollow.core.read.engine.map.HollowMapTypeReadStateShard.getAbsoluteBucketStart;

import com.netflix.hollow.core.memory.FixedLengthDataFactory;
import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsJoiner;

Expand Down Expand Up @@ -50,8 +48,8 @@ public void populateStats() {

HollowMapTypeDataElements source = from[fromIndex];

long startBucket = getAbsoluteBucketStart(source, fromOrdinal);
long endBucket = source.mapPointerAndSizeData.getElementValue((long)fromOrdinal * source.bitsPerFixedLengthMapPortion, source.bitsPerMapPointer);
long startBucket = source.getStartBucket(fromOrdinal);
long endBucket = source.getEndBucket(fromOrdinal);
long numBuckets = endBucket - startBucket;

totalOfMapBuckets += numBuckets;
Expand All @@ -77,8 +75,8 @@ public void copyRecords() {

long mapSize = 0;
if (fromOrdinal <= from[fromIndex].maxOrdinal) { // else lopsided shards resulting from skipping type shards with no additions, mapSize remains 0
long startBucket = getAbsoluteBucketStart(source, fromOrdinal);
long endBucket = source.mapPointerAndSizeData.getElementValue((long)fromOrdinal * source.bitsPerFixedLengthMapPortion, source.bitsPerMapPointer);
long startBucket = source.getStartBucket(fromOrdinal);
long endBucket = source.getEndBucket(fromOrdinal);
long numBuckets = endBucket - startBucket;

// if (false) { // SNAP: TODO: test the slow path
Expand Down
14 changes: 6 additions & 8 deletions .../main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsSplitter.java
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package com.netflix.hollow.core.read.engine.map;


import static com.netflix.hollow.core.read.engine.map.HollowMapTypeReadStateShard.getAbsoluteBucketStart;

import com.netflix.hollow.core.memory.FixedLengthDataFactory;
import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitter;

Expand Down Expand Up @@ -37,8 +35,8 @@ public void populateStats() {
int toOrdinal = ordinal >> toOrdinalShift;
to[toIndex].maxOrdinal = toOrdinal;

long startBucket = getAbsoluteBucketStart(from, ordinal);
long endBucket = from.mapPointerAndSizeData.getElementValue((long)ordinal * from.bitsPerFixedLengthMapPortion, from.bitsPerMapPointer);
long startBucket = from.getStartBucket(ordinal);
long endBucket = from.getEndBucket(ordinal);
long numBuckets = endBucket - startBucket;

shardTotalOfMapBuckets[toIndex] += numBuckets;
Expand Down Expand Up @@ -75,10 +73,10 @@ public void copyRecords() {
for(int ordinal=0;ordinal<=from.maxOrdinal;ordinal++) {
int toIndex = ordinal & toMask;
int toOrdinal = ordinal >> toOrdinalShift;
HollowMapTypeDataElements target = to[toIndex];

long startBucket = getAbsoluteBucketStart(from, ordinal);
long endBucket =from.mapPointerAndSizeData.getElementValue((long) ordinal * from.bitsPerFixedLengthMapPortion, from.bitsPerMapPointer);
HollowMapTypeDataElements target = to[toIndex];
long startBucket = from.getStartBucket(ordinal);
long endBucket = from.getEndBucket(ordinal);

// if (false) { // SNAP: TODO: test the slow path
if (target.bitsPerKeyElement == from.bitsPerKeyElement && target.bitsPerValueElement == from.bitsPerValueElement) {
Expand All @@ -88,7 +86,7 @@ public void copyRecords() {
target.entryData.copyBits(from.entryData, startBucket * bitsPerMapEntry, bucketCounter[toIndex] * bitsPerMapEntry, numBuckets * bitsPerMapEntry);
bucketCounter[toIndex] += numBuckets;
} else {
throw new RuntimeException("Unexpected for Map type");
throw new RuntimeException("Unexpected for Map type during split, expected during join"); // SNAP: TODO: remove
// for (long bucket=startBucket;bucket<endBucket;bucket++) {
// long bucketKey = from.entryData.getElementValue(bucket * from.bitsPerMapEntry, from.bitsPerKeyElement);
// long bucketValue = from.entryData.getElementValue(bucket * from.bitsPerMapEntry + from.bitsPerKeyElement, from.bitsPerValueElement);
Expand Down
Loading

0 comments on commit 5e388de

Please sign in to comment.