Skip to content

Commit

Permalink
More cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sunjeet
Sunjeet committed Aug 31, 2024
1 parent deed160 commit 35e8732
Show file tree
Hide file tree
Showing 27 changed files with 142 additions and 94 deletions.
7 changes: 7 additions & 0 deletions hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElements.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,24 @@
import com.netflix.hollow.core.memory.MemoryMode;
import com.netflix.hollow.core.memory.encoding.GapEncodedVariableLengthIntegerReader;
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
import com.netflix.hollow.core.read.HollowBlobInput;
import com.netflix.hollow.core.read.engine.map.HollowMapTypeDataElements;
import java.io.IOException;

public abstract class AbstractHollowTypeDataElements {

public int maxOrdinal;

public GapEncodedVariableLengthIntegerReader encodedAdditions;
public GapEncodedVariableLengthIntegerReader encodedRemovals;

public final ArraySegmentRecycler memoryRecycler;
public final MemoryMode memoryMode;

public AbstractHollowTypeDataElements(MemoryMode memoryMode, ArraySegmentRecycler memoryRecycler) {
this.memoryMode = memoryMode;
this.memoryRecycler = memoryRecycler;
}

public abstract void destroy();
}
20 changes: 15 additions & 5 deletions ...c/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsJoiner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,18 @@ public AbstractHollowTypeDataElementsJoiner(T[] from) {

for (AbstractHollowTypeDataElements elements : from) {
if (elements.encodedAdditions != null) {
throw new IllegalStateException("Encountered encodedAdditions in data elements splitter- this is not expected " +
throw new IllegalStateException("Encountered encodedAdditions in data elements joiner- this is not expected " +
"since encodedAdditions only exist on delta data elements and they dont carry over to target data elements, " +
"delta data elements are never split/joined");
}
}

// SNAP: TODO: should support a "too large to join" when joined ordinals will overflow
}

public T join() {

init();

initToElements();
to.maxOrdinal = -1;

populateStats();
Expand All @@ -43,13 +44,22 @@ public T join() {
}
to.encodedRemovals = GapEncodedVariableLengthIntegerReader.join(fromRemovals);

return (T) to;
return to;
}

public abstract void init();
/**
* Initialize the target data elements.
*/
public abstract void initToElements();

/**
* Populate the stats of the target data elements.
*/
public abstract void populateStats();

/**
* Copy records from the source data elements to the target data elements.
*/
public abstract void copyRecords();


Expand Down
24 changes: 19 additions & 5 deletions ...main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsSplitter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@

import com.netflix.hollow.core.memory.encoding.GapEncodedVariableLengthIntegerReader;

/**
* Join multiple {@code HollowListTypeDataElements}s into 1 {@code HollowListTypeDataElements}.
* Ordinals are remapped and corresponding data is copied over.
* The original data elements are not destroyed.
* The no. of passed data elements must be a power of 2.
*/
public abstract class AbstractHollowTypeDataElementsSplitter<T extends AbstractHollowTypeDataElements> {
public final int numSplits;
public final int toMask;
Expand All @@ -10,7 +16,7 @@ public abstract class AbstractHollowTypeDataElementsSplitter<T extends AbstractH

public T[] to;

public AbstractHollowTypeDataElementsSplitter(T from, int numSplits) {
public AbstractHollowTypeDataElementsSplitter(T from, int numSplits) { // SNAP: TODO: does this allow releasing old splits?
this.from = from;
this.numSplits = numSplits;
this.toMask = numSplits - 1;
Expand All @@ -29,8 +35,7 @@ public AbstractHollowTypeDataElementsSplitter(T from, int numSplits) {

public T[] split() {

init();

initToElements();
for(int i=0;i<to.length;i++) {
to[i].maxOrdinal = -1;
}
Expand All @@ -46,13 +51,22 @@ public T[] split() {
}
}

return (T[]) to;
return to;
}

public abstract void init();
/**
* Initialize the target data elements.
*/
public abstract void initToElements();

/**
* Populate the stats of the target data elements.
*/
public abstract void populateStats();

/**
* Copy records from the source data elements to the target data elements.
*/
public abstract void copyRecords();


Expand Down
11 changes: 6 additions & 5 deletions ...ow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElements.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public class HollowListTypeDataElements extends AbstractHollowTypeDataElements {
FixedLengthData elementData;

int bitsPerListPointer;
int bitsPerElement;
int bitsPerElement = 0;
long totalNumberOfElements = 0;

public HollowListTypeDataElements(ArraySegmentRecycler memoryRecycler) {
Expand Down Expand Up @@ -101,20 +101,21 @@ public void applyDelta(HollowListTypeDataElements fromData, HollowListTypeDataEl
new HollowListDeltaApplicator(fromData, deltaData, this).applyDelta();
}

@Override
public void destroy() {
FixedLengthDataFactory.destroy(listPointerData, memoryRecycler);
FixedLengthDataFactory.destroy(elementData, memoryRecycler);
}

public long getStartElement(int ordinal) {
return ordinal == 0 ? 0 : listPointerData.getElementValue(((long) (ordinal-1) * bitsPerListPointer), bitsPerListPointer);
long getStartElement(int ordinal) {
return ordinal == 0 ? 0 : listPointerData.getElementValue(((long)(ordinal-1) * bitsPerListPointer), bitsPerListPointer);
}

public long getEndElement(int ordinal) {
long getEndElement(int ordinal) {
return listPointerData.getElementValue((long)ordinal * bitsPerListPointer, bitsPerListPointer);
}

public void copyElementsFrom(long startElement, HollowListTypeDataElements src, long srcStartElement, long srcEndElement) {
void copyElementsFrom(long startElement, HollowListTypeDataElements src, long srcStartElement, long srcEndElement) {
if (bitsPerElement == src.bitsPerElement) {
// fast path can bulk copy elements
long numElements = srcEndElement - srcStartElement;
Expand Down
10 changes: 6 additions & 4 deletions .../main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsJoiner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public HollowListTypeDataElementsJoiner(HollowListTypeDataElements[] from) {
}

@Override
public void init() {
public void initToElements() {
this.to = new HollowListTypeDataElements(from[0].memoryMode, from[0].memoryRecycler);
}

Expand All @@ -44,14 +44,16 @@ public void populateStats() {

}
to.bitsPerListPointer = totalOfListSizes == 0 ? 1 : 64 - Long.numberOfLeadingZeros(totalOfListSizes);
to.listPointerData = FixedLengthDataFactory.get((long)to.bitsPerListPointer * (to.maxOrdinal + 1), to.memoryMode, to.memoryRecycler);
to.elementData = FixedLengthDataFactory.get((long)to.bitsPerElement * totalOfListSizes, to.memoryMode, to.memoryRecycler);
to.totalNumberOfElements = totalOfListSizes;
}

@Override
public void copyRecords() {
long elementCounter = 0;

to.listPointerData = FixedLengthDataFactory.get((long)to.bitsPerListPointer * (to.maxOrdinal + 1), to.memoryMode, to.memoryRecycler);
to.elementData = FixedLengthDataFactory.get(to.bitsPerElement * to.totalNumberOfElements, to.memoryMode, to.memoryRecycler);

for(int ordinal=0;ordinal<=to.maxOrdinal;ordinal++) {
int fromIndex = ordinal & fromMask;
int fromOrdinal = ordinal >> fromOrdinalShift;
Expand All @@ -65,7 +67,7 @@ public void copyRecords() {
to.copyElementsFrom(elementCounter, source, startElement, endElement);
elementCounter += numElements;
}
to.listPointerData.setElementValue((long) to.bitsPerListPointer * ordinal, to.bitsPerListPointer, elementCounter);
to.listPointerData.setElementValue((long)to.bitsPerListPointer * ordinal, to.bitsPerListPointer, elementCounter);
}
}
}
16 changes: 9 additions & 7 deletions ...ain/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ public HollowListTypeDataElementsSplitter(HollowListTypeDataElements from, int n
}

@Override
public void init() {
public void initToElements() {
this.to = new HollowListTypeDataElements[numSplits];
for(int i=0;i<to.length;i++) {
to[i] = new HollowListTypeDataElements(from.memoryMode, from.memoryRecycler);
Expand Down Expand Up @@ -47,12 +47,8 @@ public void populateStats() {

for(int toIndex=0;toIndex<numSplits;toIndex++) {
HollowListTypeDataElements target = to[toIndex];
target.bitsPerElement = from.bitsPerElement; // retained: it's the max across all shards in type, so splitting has no effect
target.bitsPerElement = from.bitsPerElement; // retained
target.bitsPerListPointer = maxShardTotalOfListSizes == 0 ? 1 : 64 - Long.numberOfLeadingZeros(maxShardTotalOfListSizes);

target.listPointerData = FixedLengthDataFactory.get((long)target.bitsPerListPointer * (target.maxOrdinal + 1), target.memoryMode, target.memoryRecycler);
target.elementData = FixedLengthDataFactory.get(target.bitsPerElement * totalOfListSizes[toIndex], target.memoryMode, target.memoryRecycler);

target.totalNumberOfElements = totalOfListSizes[toIndex]; // useful for heap usage stats
}
}
Expand All @@ -62,6 +58,12 @@ public void copyRecords() {
int numSplits = to.length;
long elementCounter[] = new long[numSplits];

for(int toIndex=0;toIndex<numSplits;toIndex++) {
HollowListTypeDataElements target = to[toIndex];
target.listPointerData = FixedLengthDataFactory.get((long)target.bitsPerListPointer * (target.maxOrdinal + 1), target.memoryMode, target.memoryRecycler);
target.elementData = FixedLengthDataFactory.get(target.bitsPerElement * target.totalNumberOfElements, target.memoryMode, target.memoryRecycler);
}

// count elements per split
for(int ordinal=0;ordinal<=from.maxOrdinal;ordinal++) {
int toIndex = ordinal & toMask;
Expand All @@ -75,7 +77,7 @@ public void copyRecords() {
target.copyElementsFrom(elementCounter[toIndex], from, startElement, endElement);
elementCounter[toIndex] += numElements;

target.listPointerData.setElementValue((long) target.bitsPerListPointer * toOrdinal, target.bitsPerListPointer, elementCounter[toIndex]);
target.listPointerData.setElementValue((long)target.bitsPerListPointer * toOrdinal, target.bitsPerListPointer, elementCounter[toIndex]);
}
}
}
3 changes: 2 additions & 1 deletion hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,10 @@ public HollowListTypeReadState(HollowReadStateEngine stateEngine, MemoryMode mem
this.shards = shards;
}

public HollowListTypeReadState(MemoryMode memoryMode, HollowListSchema schema, int numShards, HollowListTypeReadStateShard[] shards) {
HollowListTypeReadState(MemoryMode memoryMode, HollowListSchema schema, HollowListTypeReadStateShard[] shards) {
super(null, memoryMode, schema);
this.sampler = new HollowListSampler(schema.getName(), DisabledSamplingDirector.INSTANCE);
int numShards = shards.length;
this.shardNumberMask = numShards - 1;
this.shardOrdinalShift = 31 - Integer.numberOfLeadingZeros(numShards);

Expand Down
13 changes: 7 additions & 6 deletions hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElements.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ public class HollowMapTypeDataElements extends AbstractHollowTypeDataElements {
int bitsPerMapPointer;
int bitsPerMapSizeValue;
int bitsPerFixedLengthMapPortion;
int bitsPerKeyElement;
int bitsPerValueElement;
int bitsPerMapEntry;
int bitsPerKeyElement = 0;
int bitsPerValueElement = 0;
int bitsPerMapEntry = 0;
int emptyBucketKeyValue;
long totalNumberOfBuckets;

Expand Down Expand Up @@ -113,16 +113,17 @@ public void applyDelta(HollowMapTypeDataElements fromData, HollowMapTypeDataElem
new HollowMapDeltaApplicator(fromData, deltaData, this).applyDelta();
}

@Override
public void destroy() {
FixedLengthDataFactory.destroy(mapPointerAndSizeData, memoryRecycler);
FixedLengthDataFactory.destroy(entryData, memoryRecycler);
}

public long getStartBucket(int ordinal) {
long getStartBucket(int ordinal) {
return ordinal == 0 ? 0 : mapPointerAndSizeData.getElementValue((long)(ordinal - 1) * bitsPerFixedLengthMapPortion, bitsPerMapPointer);
}

public long getEndBucket(int ordinal) {
long getEndBucket(int ordinal) {
return mapPointerAndSizeData.getElementValue((long)ordinal * bitsPerFixedLengthMapPortion, bitsPerMapPointer);
}

Expand All @@ -134,7 +135,7 @@ int getBucketValueByAbsoluteIndex(long absoluteBucketIndex) {
return (int)entryData.getElementValue((absoluteBucketIndex * bitsPerMapEntry) + bitsPerKeyElement, bitsPerValueElement);
}

public void copyBucketsFrom(long startBucket, HollowMapTypeDataElements src, long srcStartBucket, long srcEndBucket) {
void copyBucketsFrom(long startBucket, HollowMapTypeDataElements src, long srcStartBucket, long srcEndBucket) {
if (bitsPerKeyElement == src.bitsPerKeyElement && bitsPerValueElement == src.bitsPerValueElement) {
// fast path can bulk copy buckets. emptyBucketKeyValue is same since bitsPerKeyElement is the same
long numBuckets = srcEndBucket - srcStartBucket;
Expand Down
15 changes: 8 additions & 7 deletions ...rc/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElementsJoiner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public HollowMapTypeDataElementsJoiner(HollowMapTypeDataElements[] from) {
}

@Override
public void init() {
public void initToElements() {
this.to = new HollowMapTypeDataElements(from[0].memoryMode, from[0].memoryRecycler);
}

Expand Down Expand Up @@ -59,14 +59,15 @@ public void populateStats() {
to.bitsPerMapPointer = 64 - Long.numberOfLeadingZeros(to.totalNumberOfBuckets);
to.bitsPerFixedLengthMapPortion = to.bitsPerMapSizeValue + to.bitsPerMapPointer;
to.bitsPerMapEntry = to.bitsPerKeyElement + to.bitsPerValueElement;

to.mapPointerAndSizeData = FixedLengthDataFactory.get((long)to.bitsPerFixedLengthMapPortion * (to.maxOrdinal + 1), to.memoryMode, to.memoryRecycler);
to.entryData = FixedLengthDataFactory.get((long)to.bitsPerMapEntry * to.totalNumberOfBuckets, to.memoryMode, to.memoryRecycler);
}

@Override
public void copyRecords() {
long bucketCounter = 0;

to.mapPointerAndSizeData = FixedLengthDataFactory.get((long)(to.maxOrdinal + 1) * to.bitsPerFixedLengthMapPortion, to.memoryMode, to.memoryRecycler);
to.entryData = FixedLengthDataFactory.get(to.totalNumberOfBuckets * to.bitsPerMapEntry, to.memoryMode, to.memoryRecycler);

for(int ordinal=0;ordinal<=to.maxOrdinal;ordinal++) {
int fromIndex = ordinal & fromMask;
int fromOrdinal = ordinal >> fromOrdinalShift;
Expand All @@ -82,10 +83,10 @@ public void copyRecords() {
to.copyBucketsFrom(bucketCounter, source, startBucket, endBucket);
bucketCounter += numBuckets;

mapSize = source.mapPointerAndSizeData.getElementValue((long) (fromOrdinal * source.bitsPerFixedLengthMapPortion) + source.bitsPerMapPointer, source.bitsPerMapSizeValue);
mapSize = source.mapPointerAndSizeData.getElementValue((long)(fromOrdinal * source.bitsPerFixedLengthMapPortion) + source.bitsPerMapPointer, source.bitsPerMapSizeValue);
}
to.mapPointerAndSizeData.setElementValue( (long) ordinal * to.bitsPerFixedLengthMapPortion, to.bitsPerMapPointer, bucketCounter);
to.mapPointerAndSizeData.setElementValue((long) (ordinal * to.bitsPerFixedLengthMapPortion) + to.bitsPerMapPointer, to.bitsPerMapSizeValue, mapSize);
to.mapPointerAndSizeData.setElementValue( (long)ordinal * to.bitsPerFixedLengthMapPortion, to.bitsPerMapPointer, bucketCounter);
to.mapPointerAndSizeData.setElementValue((long)(ordinal * to.bitsPerFixedLengthMapPortion) + to.bitsPerMapPointer, to.bitsPerMapSizeValue, mapSize);
}
}
}
Loading

0 comments on commit 35e8732

Please sign in to comment.