Implement sharding based on Rendezvous Hashing #238
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| package sharding | ||
|
|
||
| import ( | ||
| "hash/fnv" | ||
| "math" | ||
| ) | ||
|
|
||
| type rendezvousShardSelector struct { | ||
| keyMap map[uint64]string | ||
| weightMap map[uint64]uint32 | ||
| } | ||
|
|
||
| func hashServer(key string) uint64 { | ||
| h := fnv.New64a() | ||
| h.Write([]byte(key)) | ||
| return h.Sum64() | ||
| } | ||
|
|
||
| func NewRendezvousShardSelector(weights map[string]uint32) *rendezvousShardSelector { | ||
| weightMap := make(map[uint64]uint32, len(weights)) | ||
| keyMap := make(map[uint64]string, len(weights)) | ||
|
|
||
| for key, weight := range weights { | ||
| keyHash := hashServer(key) | ||
| keyMap[keyHash] = key | ||
| weightMap[keyHash] = weight | ||
| } | ||
| return &rendezvousShardSelector{ | ||
| keyMap: keyMap, | ||
| weightMap: weightMap, | ||
| } | ||
| } | ||
|
|
||
| func score(x uint64) float64 { | ||
| // branchless clamp to [1,MAX_UINT64-1] | ||
| x = x - ((x|-x)>>63) + (((^x)|-(^x)) >> 63) | ||
| frac := float64(x)/float64(^uint64(0)) | ||
| return 1.0/-math.Log(frac) | ||
| } | ||
|
|
||
| // PRNG without branches or allocations | ||
| func splitmix64(x uint64) uint64 { | ||
| x ^= x >> 30 | ||
| x *= 0xbf58476d1ce4e5b9 | ||
| x ^= x >> 27 | ||
| x *= 0x94d049bb133111eb | ||
| x ^= x >> 31 | ||
| return x | ||
| } | ||
|
|
||
| func (s *rendezvousShardSelector) GetShard(hash uint64) string { | ||
| var best float64 | ||
| var bestKey string | ||
|
|
||
| for keyHash, weight := range s.weightMap { | ||
| mixed := splitmix64(hash^keyHash) | ||
| current := float64(weight) * score(mixed) | ||
| if current > best { | ||
| best = current | ||
| bestKey = s.keyMap[keyHash] | ||
| } | ||
| } | ||
|
Comment on lines
+55
to
+62
There was a problem hiding this comment. Because we are simply iterating over type rendezvousShard struct {
keyHash uint64
weight uint32
key string
}
type rendezvousShardSelector struct {
shards []rendezvousShard
}Assuming this works, is it then still necessary to change ShardingBlobAccess to take a map? This function could return an index of the shard like it did before. There was a problem hiding this comment. In the unlikely event of two backends getting the same The requirement is more that the keys should be different. I think a slice should work, but during setup it might be worth checking for duplicated key hashes because loosing one storage shard without knowing why is pretty annoying. There was a problem hiding this comment. The implementation can iterate over a slice of shards, that would make the tiebreaker in case of a collision result into a stable value (biased towards earlier in the slice) and significantly reduce the changes in sharding_blob_access.go. I still think it's better to replace the interface of ShardingBlobAccessConfiguration.Shards with ShardingBlobAccessConfiguration.ShardMap since a gives a natural interface for removing and inserting new shards compared to an array. We could also error out at startup when there is a hash collision between two shards, it should never happen but maybe erroring out is preferable to ignoring an entire shard (which would be the effect of two shards sharing a hash). |
||
| return bestKey | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| package sharding_test | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/buildbarn/bb-storage/pkg/blobstore/sharding" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestRendezvousShardSelectorDistribution(t *testing.T) { | ||
| // Distribution across five backends with a total weight of 15. | ||
| weights := map[string]uint32{"a": 1, "b": 4, "c:": 2, "d": 5, "e": 3} | ||
| s := sharding.NewRendezvousShardSelector(weights) | ||
|
|
||
| // Request the shard for a very large amount of blobs | ||
| occurrences := map[string]uint32{} | ||
| for i := 0; i < 1000000; i++ { | ||
| hash := uint64(i) | ||
| occurrences[s.GetShard(hash)] += 1 | ||
| } | ||
|
|
||
| // Requests should be fanned out with a small error margin. | ||
| for shard, weight := range weights { | ||
| require.InEpsilon(t, weight*1000000/15, occurrences[shard], 0.01) | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| package sharding | ||
|
|
||
| // ShardSelector is an algorithm that for a hash resolves into a key which | ||
| // corresponds to the specific backend for that shard. | ||
| // | ||
| // The algorithm must be stable, the removal of an unavailable backend should | ||
| // not result in the reshuffling of any other blobs. | ||
| type ShardSelector interface { | ||
| GetShard(hash uint64) string | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm trying to understand the math behind this.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
However this is ieee floating point math and not real math, weight/-log(1) = -Inf which would be infinitely undesired, while while we would expect it to be infinitely desired. Thinking more about it log(0) seems to be defined as -Inf which would make the quota weight/-(-Inf) = +0 in IEEE math which is the result we actually want. If my math is correct then this means the code actually handles the extreme values in a somewhat decent manner with the MaxUint64 essentially wrapping around.
If we replace f(x) = 1/-Log(x/MaxUint64) with g(x) = 1/(Log(MaxUint64+1) - Log(x)) we should get the desired result, g(MaxUint64) = +Inf, g(0) == 0
The original proof uses log of a random uniform ]0,1[ distribution, so approximating the uniform distribution with random_uint64 / max_uint64 we could do the equivalent instead of 1.0/(log(max_uint64)-log(x))
The original proof uses the natural logarithm, log2 is just a constant away from log and we're only interested in their relative ordering rather than the score itself so we can use log2 without any difference.
There are some possible log2 bit trickery hacks, such approximating the logarithm by its integer part (i.e. number of set bits) + a polynomial that approximates its fractional part, technically I think this is the actual implementation of the the standard log algorithm just that the chosen polynomial is of high enough order to give the 15 or so bits of precision possible in float64.
Since we don't have to support the entire range of positive float64s and we don't need the full precision we could probably get something slightly faster, maybe by abusing the float64 mantissa in it's binary representation. I'll have a look if I can get something which produces sufficient precision faster.