diff --git a/pkg/kv/kvserver/asim/gossip/BUILD.bazel b/pkg/kv/kvserver/asim/gossip/BUILD.bazel index 7e6f91cead61..f491309278f6 100644 --- a/pkg/kv/kvserver/asim/gossip/BUILD.bazel +++ b/pkg/kv/kvserver/asim/gossip/BUILD.bazel @@ -14,8 +14,10 @@ go_library( "//pkg/kv/kvserver/asim/config", "//pkg/kv/kvserver/asim/state", "//pkg/roachpb", + "//pkg/settings/cluster", "//pkg/util/hlc", "//pkg/util/protoutil", + "//pkg/util/timeutil", ], ) diff --git a/pkg/kv/kvserver/asim/gossip/gossip.go b/pkg/kv/kvserver/asim/gossip/gossip.go index 36a923232863..4ef2b531a45b 100644 --- a/pkg/kv/kvserver/asim/gossip/gossip.go +++ b/pkg/kv/kvserver/asim/gossip/gossip.go @@ -20,7 +20,9 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/config" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/state" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/util/protoutil" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" ) // Gossip collects and updates the storepools of the cluster upon capacity @@ -52,7 +54,9 @@ type storeGossiper struct { addingStore bool } -func newStoreGossiper(descriptorGetter func(cached bool) roachpb.StoreDescriptor) *storeGossiper { +func newStoreGossiper( + descriptorGetter func(cached bool) roachpb.StoreDescriptor, clock timeutil.TimeSource, +) *storeGossiper { sg := &storeGossiper{ lastIntervalGossip: time.Time{}, descriptorGetter: descriptorGetter, @@ -60,7 +64,7 @@ func newStoreGossiper(descriptorGetter func(cached bool) roachpb.StoreDescriptor desc := sg.descriptorGetter(false /* cached */) knobs := kvserver.StoreGossipTestingKnobs{AsyncDisabled: true} - sg.local = kvserver.NewStoreGossip(sg, sg, knobs) + sg.local = kvserver.NewStoreGossip(sg, sg, knobs, &cluster.MakeTestingClusterSettings().SV, clock) sg.local.Ident = roachpb.StoreIdent{StoreID: desc.StoreID, NodeID: desc.Node.NodeID} return sg @@ -123,7 +127,7 @@ func (g *gossip) addStoreToGossip(s state.State, storeID state.StoreID) { g.storeGossip[storeID] = &storeGossiper{addingStore: true} g.storeGossip[storeID] = newStoreGossiper(func(cached bool) roachpb.StoreDescriptor { return s.StoreDescriptors(cached, storeID)[0] - }) + }, s.Clock()) } // Tick checks for completed gossip updates and triggers new gossip diff --git a/pkg/kv/kvserver/asim/gossip/gossip_test.go b/pkg/kv/kvserver/asim/gossip/gossip_test.go index 830482aaceec..bc45af7d3ab3 100644 --- a/pkg/kv/kvserver/asim/gossip/gossip_test.go +++ b/pkg/kv/kvserver/asim/gossip/gossip_test.go @@ -13,6 +13,7 @@ package gossip import ( "context" "testing" + "time" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator/storepool" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/config" @@ -87,11 +88,17 @@ func TestGossip(t *testing.T) { require.Len(t, gossip.exchange.pending, 0) assertStorePool(assertSameFn) + // Tick state by a large duration to ensure the below capacity changes don't + // run into the max gossip frequency limit. + storeTick := tick + // Update the usage info leases for s1 and s2, so that it exceeds the delta // required to trigger a gossip update. We do this by transferring every // lease to s2. for _, rng := range s.Ranges() { s.TransferLease(rng.RangeID(), 2) + storeTick = storeTick.Add(3 * time.Second) + s.TickClock(storeTick) } gossip.Tick(ctx, tick, s) // There should be just store 1 and 2 pending gossip updates in the exchanger. @@ -108,6 +115,10 @@ func TestGossip(t *testing.T) { // Assert that the lease counts are as expected after transferring all of // the leases to s2. require.Equal(t, int32(0), (*details[1])[1].Desc.Capacity.LeaseCount) - require.Equal(t, int32(100), (*details[1])[2].Desc.Capacity.LeaseCount) + // Depending on the capacity delta threshold, s2 may not have gossiped + // exactly when it reached 100 leases, as it earlier gossiped at 90+ leases, + // so 100 may be < lastGossip * capacityDeltaThreshold, not triggering + // gossip. Assert that the lease count gossiped is at least 90. + require.Greater(t, (*details[1])[2].Desc.Capacity.LeaseCount, int32(90)) require.Equal(t, int32(0), (*details[1])[3].Desc.Capacity.LeaseCount) } diff --git a/pkg/kv/kvserver/asim/state/impl.go b/pkg/kv/kvserver/asim/state/impl.go index 35f67860e2b7..bfdbe0b8492d 100644 --- a/pkg/kv/kvserver/asim/state/impl.go +++ b/pkg/kv/kvserver/asim/state/impl.go @@ -32,6 +32,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/spanconfig" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreporter" "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/google/btree" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/tracker" @@ -1058,6 +1059,10 @@ func (s *state) TickClock(tick time.Time) { s.clock.Set(tick.UnixNano()) } +func (s *state) Clock() timeutil.TimeSource { + return s.clock +} + // UpdateStorePool modifies the state of the StorePool for the Store with // ID StoreID. func (s *state) UpdateStorePool( diff --git a/pkg/kv/kvserver/asim/state/state.go b/pkg/kv/kvserver/asim/state/state.go index 9251da3c1f71..058071d64766 100644 --- a/pkg/kv/kvserver/asim/state/state.go +++ b/pkg/kv/kvserver/asim/state/state.go @@ -167,6 +167,8 @@ type State interface { ClusterUsageInfo() *ClusterUsageInfo // TickClock modifies the state Clock time to Tick. TickClock(time.Time) + // Clock returns the state Clock. + Clock() timeutil.TimeSource // UpdateStorePool modifies the state of the StorePool for the Store with // ID StoreID. UpdateStorePool(StoreID, map[roachpb.StoreID]*storepool.StoreDetail) @@ -290,6 +292,8 @@ type ManualSimClock struct { nanos int64 } +var _ timeutil.TimeSource = &ManualSimClock{} + // Now returns the current time. func (m *ManualSimClock) Now() time.Time { return timeutil.Unix(0, m.nanos) @@ -300,6 +304,18 @@ func (m *ManualSimClock) Set(tsNanos int64) { m.nanos = tsNanos } +func (m *ManualSimClock) Since(t time.Time) time.Duration { + return m.Now().Sub(t) +} + +func (m *ManualSimClock) NewTimer() timeutil.TimerI { + panic("unimplemented") +} + +func (m *ManualSimClock) NewTicker(duration time.Duration) timeutil.TickerI { + panic("unimplemented") +} + // Keys in the simulator are 64 bit integers. They are mapped to Keys in // cockroach as the decimal representation, with 0 padding such that they are // lexicographically ordered as strings. The simplification to limit keys to diff --git a/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_add_node b/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_add_node index 97ee13f8f0a6..ddb1c596b52d 100644 --- a/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_add_node +++ b/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_add_node @@ -42,14 +42,14 @@ plot stat=replicas sample=1 261 ┤ ╭╭──╯ 241 ┤ ╭╭─╯ 221 ┤ ╭───╯ - 201 ┤ ╭╭─╯ - 181 ┤ ╭──╯ - 161 ┤ ╭──╯ + 201 ┤ ╭──╯ + 181 ┤ ╭──╯╯ + 161 ┤ ╭─╯─╯ 140 ┤ ╭──╯╯ 120 ┤ ╭─╯╯ 100 ┤ ╭──╯ - 80 ┤ ╭─╯╯ - 60 ┤ ╭──╯ + 80 ┤ ╭──╯╯ + 60 ┤ ╭─╯╯ 40 ┤ ╭─╯ 20 ┤ ╭──╯ 0 ┼─╯ diff --git a/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_fulldisk b/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_fulldisk index 6654796c7829..3ec34dc326e7 100644 --- a/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_fulldisk +++ b/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_fulldisk @@ -21,42 +21,43 @@ plot stat=replicas ---- ---- - 336 ┤ ╭╮ ╭╭╮╭╮─╭╮╭╭╮ ╭╮╭──╮╮╭╭─ - 325 ┤ ╭╮╭────╮╭────────╯╰╯╰─╯╰─╯╰──────────────╮╭╯─╯╰──╯╯ - 314 ┤ ╭╭╭╮╭─╭──╯╰╯╯╰╯╰╰╯ ╰╯ ╰╯ ╰╯╰╯╰╯ ╰╯ - 302 ┼───────────────╮─────────╯ - 291 ┤ ╰───╮ - 280 ┤ ╰╮ ╭╮ - 269 ┤ ╰─╯╰╮ - 258 ┤ ╰╮ - 246 ┤ ╰──╮ - 235 ┤ ╰╮ - 224 ┤ ╰╮ - 213 ┤ ╰╮╭────╮ ╭╮ - 202 ┤ ╰╯ ╰╮ ╭──────────╯╰───╮ - 190 ┤ ╰─────╮ ╭───╮ ╭╯ │ - 179 ┤ ╰───╯ ╰─╯ ╰─╮ ╭──╮ - 168 ┤ ╰─╯ ╰ + 342 ┤ ╭╮╭╮ ╭╮╭╮ + 330 ┤ ╭╭╮╭╭╭───────────╯╰───╯╰──────╮╭────────╮╭─╮╭─── + 318 ┤ ╭╮╭──────╯╰──╯─╯╰─╯╯ ╰╯ ╰╯ ╰╯ ╰╯╰─╯╰╯╯╰╯╰╯ ╰╯ + 307 ┤╭────────────────────╭─╯╰╯╯╰╯ + 295 ┼─────────────────╮───╯╰╯ + 283 ┤ ╰──╮╭─╮ + 271 ┤ ╰╯ ╰╮ + 259 ┤ │ + 248 ┤ ╰─╮ + 236 ┤ ╰─╮ + 224 ┤ ╰──╮ + 212 ┤ ╰───╮ + 200 ┤ ╰──╮ ╭──╮ ╭╮ + 189 ┤ ╰───╮ ╭╮│ ╰─────╮│╰───╮ ╭ + 177 ┤ ╰────╮╭────╮ ╭───╯╰╯ ╰╯ ╰─╯ + 165 ┤ ╰╯ ╰─╯ replicas ---- ---- + # Plot the % of disk storage capacity used. We should see s5 hovering right # around 92.5-95% (the storage capacity threshold value). plot stat=disk_fraction_used ---- ---- - 0.98 ┤ ╭─╮ ╭╮ ╭╮╭─╮╭──╮ ╭──────╮╭─╮ ╭───╮ ╭╮ ╭╮╭─╮ ╭───╮ ╭─╮ - 0.91 ┤ ╭───────╯ ╰─╯╰──╯╰╯ ╰╯ ╰──╯ ╰╯ ╰────╯ ╰────╯╰──╯╰╯ ╰──╯ ╰───╯ ╰ - 0.85 ┼──────╯ - 0.78 ┤ - 0.72 ┤ - 0.65 ┤ - 0.59 ┤ - 0.52 ┤ + 0.99 ┤ ╭─╮ ╭╮ ╭╮ ╭─╮╭ + 0.93 ┤ ╭────────────╯ ╰───────────────────────────╯╰──────╮╭────────╮│╰──╯ ╰╯ + 0.86 ┼╮╭───────╯ ╰╯ ╰╯ + 0.79 ┤╰╯ + 0.73 ┤ + 0.66 ┤ + 0.60 ┤ + 0.53 ┤ 0.46 ┤ - 0.39 ┤ + 0.40 ┤ 0.33 ┤ 0.26 ┤ 0.20 ┤ diff --git a/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_multi_store b/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_multi_store index de8f4766e5da..50a65a544c49 100644 --- a/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_multi_store +++ b/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_multi_store @@ -27,20 +27,20 @@ plot stat=leases 14.00 ┼╮ 13.07 ┤╰╮ - 12.13 ┤ ╰╮ - 11.20 ┤ │ + 12.13 ┤ │ + 11.20 ┤ ╰╮ 10.27 ┤ │ 9.33 ┤ │ 8.40 ┤ ╰╮ - 7.47 ┤ ╰╮ - 6.53 ┤ │ - 5.60 ┤ │ - 4.67 ┤ │ - 3.73 ┤ ╰───────────╮ - 2.80 ┤ │ - 1.87 ┤ ╭───────────╮──────────────╮ - 0.93 ┤╭╭╭──────────────────────────────────────────────────────────────────────────── - 0.00 ┼──╯─────────────╯──────────────╯ + 7.47 ┤ │ + 6.53 ┤ │ + 5.60 ┤ │ + 4.67 ┤ ╰╮ + 3.73 ┤ │ + 2.80 ┤ ╰───────────╮ + 1.87 ┤ ╭─────────────╮──────────────╮ + 0.93 ┤╭╭╮╭─────────────────────────────────────────────────────────────────────────── + 0.00 ┼─╯╰╯╯───────────╯──────────────╯ leases ---- ---- diff --git a/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_rebalancing b/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_rebalancing index c8d99474b0d8..a7552e472751 100644 --- a/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_rebalancing +++ b/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_rebalancing @@ -75,6 +75,17 @@ setting gossip_delay=20s # assertion is reached due to how the system reacts to the long gossip delays. eval duration=5m samples=2 seed=42 ---- +failed assertion sample 1 + balance stat=qps threshold=(<1.15) ticks=6 + max/mean=1.17 tick=0 + max/mean=1.17 tick=1 + max/mean=1.17 tick=2 + max/mean=1.17 tick=3 + max/mean=1.40 tick=4 + max/mean=1.41 tick=5 + steady state stat=qps threshold=(<0.05) ticks=6 + store=1 min/mean=1.00 max/mean=6.00 + store=4 min/mean=1.00 max/mean=0.76 failed assertion sample 2 balance stat=qps threshold=(<1.15) ticks=6 max/mean=2.00 tick=0 @@ -95,18 +106,18 @@ plot stat=qps sample=3 6533 ┤ │ │ 6067 ┤ │ │ 5600 ┤ │ ╰╮ - 5133 ┤ │ │ ╭───╮ - 4667 ┤ │ │ │ │ - 4200 ┤ │ │ ╭───╮ ╭───╮ │ │ - 3733 ┤ │ │ │ │ │ │ │ │ - 3267 ┤ │ │ │ │ │ │ │ │ - 2800 ┤ │ │╭╭─────╮╭╯ │ ╭╯ ╰╮ ╭──╮ ╭─────╮ - 2333 ┤ │ │││ │││ │ │ │╭╯ │ │ │ - 1867 ┤ │ ╰││ │╰╮╭───╮╮│ ││╭──╰╮╭───────────╮╮╭─────────────╮ - 1400 ┤ │ ││ ││╭╯ │││ │││ ╭╯│ │ │││ │ - 933 ┤ │ ╭╭╯ ╰╮│╭────────────────────────────────────────────────────────────── - 467 ┤ │ ││ │││ ││ ││││ ││ │ ││ │ - 0 ┼────────────────╯────────────────────────────────╯───────────────╯ + 5133 ┤ │ │ + 4667 ┤ │ │ + 4200 ┤ │ │ ╭────╮ ╭──╮ ╭─────╮ + 3733 ┤ │ │ │ │ │ │ │ │ + 3267 ┤ │ │ │ │ │ │ │ │ + 2800 ┤ │ │╭╭─────╮ │ │ ╭─╮ ╭─────╮ │ ╰───────╮ + 2333 ┤ │ │││ ││╭╯ ╰╮ │ │ │ │ │ │ + 1867 ┤ │ ╰││ ╰╰╮ ╭╭─────────╮───────────────╮ │ ╭──────╰╮╭──╮ ╭─────╮ + 1400 ┤ │ ││ ││ ││ ││ │╰╮│ │ │ │ │ │ ││ │ │ │ + 933 ┤ │ ╭╭╯ ││╭──────────────────────────────╮╮│──╮──╭─────────╭───────────── + 467 ┤ │ ││ ││││ ││ │ │ ││ ││ │││ │╰╮│ │ │ │ │ │ + 0 ┼────────────────╯──────────────────────────────╰─────────────────╯────╰───────╯ qps ---- ---- @@ -118,22 +129,22 @@ plot stat=replica_moves sample=3 ---- ---- - 22.00 ┤ ╭────────────────────────────────────── - 20.53 ┤ ╭────╭─────────────────────────────────────── - 19.07 ┤ ╭╯ │ - 17.60 ┤ ╭─────╭──────────╯ - 16.13 ┤ ╭╯ │ - 14.67 ┤ ╭╯ ╭╯ ╭──────────────────────────────────────────── - 13.20 ┤ │ │ │ - 11.73 ┤ │ │ ╭╯ - 10.27 ┤ ╭─────╭──────╯ ╭───╯ - 8.80 ┤ ╭╭─────╯ ╭╯ - 7.33 ┤ ╭─────╭╯╭────────────╯ ╭───────╭────────────────────────────── - 5.87 ┤ ╭╯ │╭╯ ╭────────╯ - 4.40 ┤ │ ╭───╭╯ ╭─────╭──────────╯ ╭───────────────────────────── - 2.93 ┤ ╭╯ ╭╯ ╭╯ ╭─╯ ╭╯ ╭─╯ - 1.47 ┤ │╭─╭───╯ ╭╭──────╯───────────────────╯╭────────────────────────────── - 0.00 ┼────────────────────╯───────────────────────────╯ + 23.00 ┤ ╭ + 21.47 ┤ ╭─────────────────╯ + 19.93 ┤ ╭──────╯ + 18.40 ┤ ╭────────────╯ + 16.87 ┤ ╭╯ + 15.33 ┤ ╭─────────────────╯ + 13.80 ┤ ╭─╯ + 12.27 ┤ ╭─╯ ╭──────────────────────────╭────────── + 10.73 ┤ ╭─╭────────────────────────╯ ╭─────────────────────╯ ╭╭─ + 9.20 ┤ ╭╯╭╯ ╭─────────────╯ ╭╭───────╭─ + 7.67 ┤ ╭─────╭─╯ ╭──────╯ ╭─────────────────────────────╯ + 6.13 ┤ ╭╯ │ ╭─────────╯ ╭─────╭╯╯────╯ │ + 4.60 ┤ │ ╭───╯╭╯ ╭─╭──────────────╯ ╭──────╯ + 3.07 ┤ ╭╯ ╭╯ ╭╯ ╭─────╯ ╭╯ + 1.53 ┤ │╭─╭────╯ ╭─────╯╯ │ + 0.00 ┼───────────────────╯─╯───────────────────────────────╯ replica_moves ---- ---- diff --git a/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_zone_config b/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_zone_config index 14e8d6753460..7d714291303f 100644 --- a/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_zone_config +++ b/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_zone_config @@ -55,22 +55,22 @@ plot stat=replicas ---- ---- - 52.00 ┤ ╭╭──────────────────────────────────────────────────────────── - 48.53 ┤ ╭╭───────────────────────────────────────────────────────────────── - 45.07 ┤ ╭╮││─╯╯ - 41.60 ┤ ╭│╭─╯╯ - 38.13 ┤ │││╯ - 34.67 ┤ ╭╭╯ - 31.20 ┤ ╭╭╯ - 27.73 ┤ ││╯ - 24.27 ┤ ││ - 20.80 ┤ ╭╭╯ - 17.33 ┼────────╮╮ - 13.87 ┤ ╰╰╮╮╮ - 10.40 ┤ ╰╰─╮╮ - 6.93 ┤ ╰╰╰──╮ - 3.47 ┤ ╰╰╰╰─╰────╮ - 0.00 ┤ ╰╰─╰╰────╰─────────────────────────────────────────────────────────── + 52.34 ┤ ╭╮ ╭──────────────────────────────────────────────────────────── + 48.85 ┤ ╭╮╭╮╭╭────────────────────────────────────────────────────────────── + 45.36 ┤ ╭╮ │╭│╰─╯─╯╯ + 41.87 ┤ │╰╭╭╮│╯ + 38.38 ┤ │ ││╰╯ + 34.89 ┤ │ ││ + 31.41 ┤ ╭╭─╯ + 27.92 ┤ │││ + 24.43 ┤ ╭│╯ + 20.94 ┤ ╭╭╯ + 17.45 ┼────────╮╮ + 13.96 ┤ ╰╰╮╮ + 10.47 ┤ ╰╰─╮╮ + 6.98 ┤ ╰╰─╰╮╮ + 3.49 ┤ ╰╰─╰────╮─╮─╮ + 0.00 ┤ ╰──╰╰──╰───────────────────────────────────────────────────────────── replicas ---- ---- diff --git a/pkg/kv/kvserver/asim/tests/testdata/rand/rand_ranges b/pkg/kv/kvserver/asim/tests/testdata/rand/rand_ranges index 783671ee4bf1..8ea93e94fdd4 100644 --- a/pkg/kv/kvserver/asim/tests/testdata/rand/rand_ranges +++ b/pkg/kv/kvserver/asim/tests/testdata/rand/rand_ranges @@ -178,7 +178,7 @@ configurations generated using seed 2643318057788968173 basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 number of mutation events=0, number of assertion events=0 initial state at 2022-03-21 11:00:00: - stores(15)=[s1n1=(replicas(65)),s2n2=(replicas(64)),s3n3=(replicas(66)),s4n4=(replicas(63)),s5n5=(replicas(63)),s6n6=(replicas(63)),s7n7=(replicas(63)),s8n8=(replicas(62)),s9n9=(replicas(64)),s10n10=(replicas(64)),s11n11=(replicas(62)),s12n12=(replicas(63)),s13n13=(replicas(63)),s14n14=(replicas(64)),s15n15=(replicas(64))] + stores(15)=[s1n1=(replicas(68)),s2n2=(replicas(66)),s3n3=(replicas(67)),s4n4=(replicas(67)),s5n5=(replicas(66)),s6n6=(replicas(67)),s7n7=(replicas(65)),s8n8=(replicas(70)),s9n9=(replicas(69)),s10n10=(replicas(65)),s11n11=(replicas(69)),s12n12=(replicas(68)),s13n13=(replicas(65)),s14n14=(replicas(64)),s15n15=(replicas(68))] topology: US US_1 @@ -190,17 +190,68 @@ US no events were scheduled sample2: failed assertion conformance unavailable=0 under=0 over=0 violating=0 - actual unavailable=0 under=0, over=9 violating=0 + actual unavailable=0 under=0, over=60 violating=0 over replicated: - r120:000001{8921-9080} [(n8,s8):2, (n15,s15):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 - r133:000002{0988-1147} [(n3,s3):2, (n5,s5):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 - r138:0000021{783-942} [(n3,s3):2, (n12,s12):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 - r156:0000024{645-804} [(n3,s3):2, (n2,s2):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 - r243:0000038{478-637} [(n3,s3):2, (n12,s12):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 - r322:0000051{039-198} [(n1,s1):2, (n3,s3):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 - r567:00000{89994-90153} [(n3,s3):2, (n5,s5):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 - r605:0000096{036-195} [(n3,s3):3, (n8,s8):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 - r875:000013{8966-9125} [(n3,s3):3, (n2,s2):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r14:0000002{067-226} [(n13,s13):3, (n1,s1):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r30:0000004{611-770} [(n13,s13):4, (n15,s15):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r41:0000006{360-519} [(n13,s13):3, (n9,s9):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r82:000001{2879-3038} [(n2,s2):3, (n13,s13):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r100:0000015{741-900} [(n13,s13):4, (n1,s1):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r111:0000017{490-649} [(n15,s15):3, (n13,s13):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r115:0000018{126-285} [(n13,s13):3, (n11,s11):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r129:0000020{352-511} [(n13,s13):3, (n6,s6):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r132:0000020{829-988} [(n13,s13):3, (n10,s10):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r154:0000024{327-486} [(n13,s13):3, (n10,s10):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r177:000002{7984-8143} [(n4,s4):3, (n14,s14):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r186:0000029{415-574} [(n13,s13):3, (n12,s12):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r188:0000029{733-892} [(n13,s13):3, (n10,s10):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r205:0000032{436-595} [(n13,s13):5, (n7,s7):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r214:000003{3867-4026} [(n13,s13):5, (n8,s8):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r218:0000034{503-662} [(n9,s9):2, (n13,s13):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r281:0000044{520-679} [(n13,s13):2, (n4,s4):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r288:0000045{633-792} [(n13,s13):2, (n7,s7):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r314:0000049{767-926} [(n5,s5):2, (n14,s14):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r345:0000054{696-855} [(n13,s13):2, (n11,s11):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r352:0000055{809-968} [(n13,s13):3, (n8,s8):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r357:0000056{604-763} [(n13,s13):3, (n15,s15):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r368:0000058{353-512} [(n5,s5):4, (n12,s12):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r384:000006{0897-1056} [(n4,s4):2, (n13,s13):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r404:0000064{077-236} [(n13,s13):5, (n6,s6):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r412:0000065{349-508} [(n9,s9):4, (n13,s13):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r428:000006{7893-8052} [(n13,s13):2, (n15,s15):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r429:0000068{052-211} [(n13,s13):2, (n11,s11):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r449:0000071{232-391} [(n13,s13):2, (n7,s7):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r468:0000074{253-412} [(n13,s13):2, (n2,s2):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r489:0000077{592-751} [(n13,s13):2, (n6,s6):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r490:0000077{751-910} [(n15,s15):4, (n13,s13):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r509:0000080{772-931} [(n2,s2):3, (n13,s13):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r537:0000085{224-383} [(n13,s13):2, (n7,s7):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r572:0000090{789-948} [(n13,s13):2, (n14,s14):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r586:0000093{015-174} [(n13,s13):3, (n5,s5):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r588:0000093{333-492} [(n13,s13):2, (n6,s6):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r608:0000096{513-672} [(n13,s13):2, (n8,s8):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r616:0000097{785-944} [(n13,s13):3, (n5,s5):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r620:0000098{421-580} [(n13,s13):3, (n9,s9):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r665:0000105{576-735} [(n13,s13):2, (n8,s8):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r706:0000112{095-254} [(n13,s13):3, (n6,s6):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r725:0000115{116-275} [(n14,s14):2, (n3,s3):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r752:0000119{409-568} [(n13,s13):4, (n14,s14):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r753:0000119{568-727} [(n13,s13):3, (n10,s10):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r760:0000120{681-840} [(n13,s13):5, (n4,s4):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r761:0000120{840-999} [(n13,s13):2, (n10,s10):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r785:0000124{656-815} [(n13,s13):2, (n14,s14):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r792:0000125{769-928} [(n13,s13):2, (n11,s11):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r795:0000126{246-405} [(n13,s13):4, (n5,s5):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r799:000012{6882-7041} [(n13,s13):2, (n9,s9):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r811:0000128{790-949} [(n13,s13):2, (n4,s4):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r813:0000129{108-267} [(n13,s13):2, (n11,s11):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r825:0000131{016-175} [(n13,s13):2, (n12,s12):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r885:0000140{556-715} [(n13,s13):3, (n4,s4):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r895:0000142{146-305} [(n13,s13):2, (n11,s11):3] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r913:0000145{008-167} [(n13,s13):3, (n7,s7):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r923:0000146{598-757} [(n13,s13):3, (n5,s5):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r940:0000149{301-460} [(n13,s13):3, (n10,s10):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r943:0000149{778-937} [(n13,s13):3, (n2,s2):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 ---------------------------------- sample3: start running configurations generated using seed 6972490225919430754 @@ -209,5 +260,109 @@ configurations generated using seed 6972490225919430754 randomized ranges with placement_type=random, ranges=479, key_space=199954, replication_factor=1, bytes=0 basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 number of mutation events=0, number of assertion events=0 -sample3: pass +initial state at 2022-03-21 11:00:00: + stores(15)=[s1n1=(replicas(39)),s2n2=(replicas(41)),s3n3=(replicas(36)),s4n4=(replicas(39)),s5n5=(replicas(33)),s6n6=(replicas(42)),s7n7=(replicas(35)),s8n8=(replicas(40)),s9n9=(replicas(36)),s10n10=(replicas(37)),s11n11=(replicas(36)),s12n12=(replicas(40)),s13n13=(replicas(40)),s14n14=(replicas(39)),s15n15=(replicas(36))] +topology: +US + US_1 + └── [1 2 3 4 5] + US_2 + └── [6 7 8 9 10] + US_3 + └── [11 12 13 14 15] +no events were scheduled +sample3: failed assertion + conformance unavailable=0 under=0 over=0 violating=0 + actual unavailable=0 under=0, over=90 violating=0 +over replicated: + r5:000000{1668-2085} [(n10,s10):5, (n4,s4):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r7:0000002{502-919} [(n5,s5):5, (n13,s13):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r18:0000007{089-506} [(n1,s1):3, (n12,s12):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r23:0000009{174-591} [(n2,s2):7, (n12,s12):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r24:00000{09591-10008} [(n8,s8):7, (n14,s14):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r31:0000012{510-927} [(n8,s8):4, (n15,s15):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r37:0000015{012-429} [(n13,s13):7, (n12,s12):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r40:0000016{263-680} [(n10,s10):4, (n15,s15):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r44:000001{7931-8348} [(n3,s3):5, (n15,s15):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r56:000002{2935-3352} [(n8,s8):5, (n6,s6):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r57:0000023{352-769} [(n8,s8):6, (n2,s2):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r62:0000025{437-854} [(n15,s15):5, (n7,s7):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r67:0000027{522-939} [(n1,s1):7, (n5,s5):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r71:0000029{190-607} [(n8,s8):4, (n2,s2):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r82:000003{3777-4194} [(n10,s10):8, (n2,s2):9] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r83:0000034{194-611} [(n10,s10):6, (n5,s5):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r85:0000035{028-445} [(n6,s6):7, (n5,s5):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r98:0000040{449-866} [(n15,s15):4, (n5,s5):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r106:000004{3785-4202} [(n3,s3):7, (n14,s14):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r108:000004{4619-5036} [(n5,s5):6, (n2,s2):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r124:0000051{291-708} [(n3,s3):5, (n14,s14):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r127:0000052{542-959} [(n8,s8):5, (n6,s6):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r128:000005{2959-3376} [(n8,s8):6, (n1,s1):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r136:0000056{295-712} [(n13,s13):8, (n12,s12):9] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r137:000005{6712-7129} [(n7,s7):5, (n5,s5):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r138:0000057{129-546} [(n8,s8):6, (n12,s12):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r155:0000064{218-635} [(n4,s4):5, (n5,s5):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r160:0000066{303-720} [(n1,s1):6, (n8,s8):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r161:000006{6720-7137} [(n5,s5):5, (n2,s2):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r162:0000067{137-554} [(n5,s5):5, (n1,s1):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r163:0000067{554-971} [(n8,s8):6, (n3,s3):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r169:0000070{056-473} [(n7,s7):6, (n8,s8):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r173:000007{1724-2141} [(n8,s8):8, (n12,s12):9] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r180:000007{4643-5060} [(n8,s8):3, (n13,s13):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r186:0000077{145-562} [(n5,s5):6, (n6,s6):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r188:000007{7979-8396} [(n5,s5):5, (n14,s14):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r194:0000080{481-898} [(n7,s7):7, (n8,s8):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r205:0000085{068-485} [(n3,s3):4, (n10,s10):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r207:000008{5902-6319} [(n11,s11):6, (n8,s8):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r209:000008{6736-7153} [(n8,s8):5, (n5,s5):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r212:000008{7987-8404} [(n4,s4):4, (n14,s14):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r215:0000089{238-655} [(n8,s8):3, (n15,s15):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r221:000009{1740-2157} [(n7,s7):4, (n6,s6):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r233:000009{6744-7161} [(n3,s3):5, (n10,s10):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r237:0000098{412-829} [(n8,s8):3, (n3,s3):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r246:0000102{165-582} [(n3,s3):6, (n15,s15):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r262:000010{8837-9254} [(n3,s3):5, (n15,s15):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r264:00001{09671-10088} [(n6,s6):6, (n1,s1):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r265:0000110{088-505} [(n11,s11):4, (n5,s5):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r274:000011{3841-4258} [(n8,s8):5, (n14,s14):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r280:0000116{343-760} [(n13,s13):7, (n3,s3):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r281:000011{6760-7177} [(n3,s3):5, (n13,s13):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r283:000011{7594-8011} [(n4,s4):4, (n5,s5):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r293:000012{1764-2181} [(n10,s10):3, (n1,s1):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r297:0000123{432-849} [(n15,s15):6, (n8,s8):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r299:0000124{266-683} [(n4,s4):8, (n5,s5):9] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r317:000013{1772-2189} [(n8,s8):6, (n13,s13):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r319:000013{2606-3023} [(n1,s1):4, (n4,s4):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r323:0000134{274-691} [(n8,s8):7, (n13,s13):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r324:000013{4691-5108} [(n6,s6):7, (n1,s1):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r325:0000135{108-525} [(n5,s5):6, (n6,s6):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r335:0000139{278-695} [(n1,s1):6, (n12,s12):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r344:0000143{031-448} [(n1,s1):4, (n5,s5):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r352:0000146{367-784} [(n6,s6):5, (n12,s12):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r353:000014{6784-7201} [(n3,s3):6, (n4,s4):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r371:0000154{290-707} [(n8,s8):6, (n5,s5):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r378:0000157{209-626} [(n5,s5):6, (n6,s6):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r381:0000158{460-877} [(n15,s15):6, (n8,s8):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r387:000016{0962-1379} [(n5,s5):8, (n1,s1):9] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r392:0000163{047-464} [(n8,s8):5, (n3,s3):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r393:0000163{464-881} [(n6,s6):4, (n15,s15):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r396:000016{4715-5132} [(n5,s5):7, (n2,s2):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r399:000016{5966-6383} [(n8,s8):5, (n1,s1):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r400:0000166{383-800} [(n8,s8):6, (n3,s3):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r402:0000167{217-634} [(n3,s3):5, (n11,s11):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r404:0000168{051-468} [(n3,s3):5, (n2,s2):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r412:0000171{387-804} [(n12,s12):4, (n2,s2):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r421:0000175{140-557} [(n8,s8):7, (n15,s15):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r425:000017{6808-7225} [(n6,s6):4, (n13,s13):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r434:0000180{561-978} [(n6,s6):5, (n12,s12):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r435:000018{0978-1395} [(n8,s8):3, (n5,s5):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r439:000018{2646-3063} [(n1,s1):4, (n12,s12):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r440:0000183{063-480} [(n3,s3):5, (n2,s2):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r447:000018{5982-6399} [(n3,s3):7, (n11,s11):8] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r448:0000186{399-816} [(n7,s7):5, (n5,s5):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r461:000019{1820-2237} [(n6,s6):5, (n1,s1):6] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r463:000019{2654-3071} [(n4,s4):3, (n5,s5):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r465:0000193{488-905} [(n7,s7):4, (n1,s1):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r466:000019{3905-4322} [(n15,s15):4, (n1,s1):5] applying ttl_seconds=0 num_replicas=1 num_voters=1 + r472:0000196{407-824} [(n7,s7):6, (n15,s15):7] applying ttl_seconds=0 num_replicas=1 num_voters=1 ---------------------------------- diff --git a/pkg/kv/kvserver/asim/tests/testdata/rand/weighted_rand b/pkg/kv/kvserver/asim/tests/testdata/rand/weighted_rand index c9d3fc04fad7..0721a1154099 100644 --- a/pkg/kv/kvserver/asim/tests/testdata/rand/weighted_rand +++ b/pkg/kv/kvserver/asim/tests/testdata/rand/weighted_rand @@ -73,7 +73,7 @@ configurations generated using seed 5571782338101878760 basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 number of mutation events=0, number of assertion events=0 initial state at 2022-03-21 11:00:00: - stores(6)=[s1n1=(replicas(132)),s2n1=(replicas(130)),s3n2=(replicas(131)),s4n2=(replicas(437)),s5n3=(replicas(428)),s6n3=(replicas(431))] + stores(6)=[s1n1=(replicas(131)),s2n1=(replicas(128)),s3n2=(replicas(130)),s4n2=(replicas(438)),s5n3=(replicas(432)),s6n3=(replicas(430))] sample1: pass ---------------------------------- sample2: start running @@ -83,7 +83,7 @@ configurations generated using seed 4299969443970870044 basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 number of mutation events=0, number of assertion events=0 initial state at 2022-03-21 11:00:00: - stores(6)=[s1n1=(replicas(140)),s2n1=(replicas(136)),s3n2=(replicas(139)),s4n2=(replicas(169)),s5n3=(replicas(142)),s6n3=(replicas(171))] + stores(6)=[s1n1=(replicas(145)),s2n1=(replicas(140)),s3n2=(replicas(139)),s4n2=(replicas(166)),s5n3=(replicas(142)),s6n3=(replicas(165))] sample2: pass ---------------------------------- sample3: start running @@ -93,6 +93,6 @@ configurations generated using seed 4157513341729910236 basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 number of mutation events=0, number of assertion events=0 initial state at 2022-03-21 11:00:00: - stores(6)=[s1n1=(replicas(180)),s2n1=(replicas(181)),s3n2=(replicas(183)),s4n2=(replicas(386)),s5n3=(replicas(247)),s6n3=(replicas(386))] + stores(6)=[s1n1=(replicas(190)),s2n1=(replicas(174)),s3n2=(replicas(182)),s4n2=(replicas(386)),s5n3=(replicas(247)),s6n3=(replicas(384))] sample3: pass ---------------------------------- diff --git a/pkg/kv/kvserver/client_split_test.go b/pkg/kv/kvserver/client_split_test.go index 4611f7103c52..a9b5965fcc8b 100644 --- a/pkg/kv/kvserver/client_split_test.go +++ b/pkg/kv/kvserver/client_split_test.go @@ -2347,7 +2347,13 @@ func TestStoreRangeGossipOnSplits(t *testing.T) { overrideCapacityFraction := 0.5 ctx := context.Background() + + // Override the store gossip frequency to zero, so that gossip always + // triggers on capacity changes. + st := cluster.MakeTestingClusterSettings() + kvserver.MaxStoreGossipFrequency.Override(ctx, &st.SV, 0*time.Millisecond) s := serverutils.StartServerOnly(t, base.TestServerArgs{ + Settings: st, Knobs: base.TestingKnobs{ Store: &kvserver.StoreTestingKnobs{ DisableMergeQueue: true, diff --git a/pkg/kv/kvserver/replicate_queue_test.go b/pkg/kv/kvserver/replicate_queue_test.go index 86314492334e..940e79c99880 100644 --- a/pkg/kv/kvserver/replicate_queue_test.go +++ b/pkg/kv/kvserver/replicate_queue_test.go @@ -250,6 +250,11 @@ func TestReplicateQueueRebalanceMultiStore(t *testing.T) { st := server.ClusterSettings() st.Manual.Store(true) allocatorimpl.LeaseRebalanceThreshold.Override(ctx, &st.SV, leaseRebalanceThreshold) + // We speed up replicate queue processing (scan min/max idle) time, + // this causes actions to occur more frequently than in practice and + // ultimately this test will fail unless we correspondingly increase + // the max store gossip frequency. + kvserver.MaxStoreGossipFrequency.Override(ctx, &st.SV, 0) } // Add a few ranges per store. diff --git a/pkg/kv/kvserver/store.go b/pkg/kv/kvserver/store.go index dabcd951c92e..e6fa64bd0dbb 100644 --- a/pkg/kv/kvserver/store.go +++ b/pkg/kv/kvserver/store.go @@ -1625,7 +1625,7 @@ func NewStore( updateSystemConfigUpdateQueueLimits) if s.cfg.Gossip != nil { - s.storeGossip = NewStoreGossip(cfg.Gossip, s, cfg.TestingKnobs.GossipTestingKnobs) + s.storeGossip = NewStoreGossip(cfg.Gossip, s, cfg.TestingKnobs.GossipTestingKnobs, &cfg.Settings.SV, timeutil.DefaultTimeSource{}) // Add range scanner and configure with queues. s.scanner = newReplicaScanner( diff --git a/pkg/kv/kvserver/store_gossip.go b/pkg/kv/kvserver/store_gossip.go index b555bc32eb4b..52a223b7cb79 100644 --- a/pkg/kv/kvserver/store_gossip.go +++ b/pkg/kv/kvserver/store_gossip.go @@ -21,21 +21,40 @@ import ( "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/retry" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/cockroachdb/cockroach/pkg/util/syncutil" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/errors" "github.com/cockroachdb/redact" ) +var GossipWhenCapacityDeltaExceedsFraction = settings.RegisterFloatSetting( + settings.SystemOnly, + "kv.store_gossip.capacity_delta_threshold", + "the fraction from the last gossiped store capacity values which need be "+ + "exceeded before the store will gossip immediately without waiting for "+ + "the periodic gossip interval, at most kv.store_gossip.max_frequency "+ + "frequency", + defaultGossipWhenCapacityDeltaExceedsFraction, +) + +var MaxStoreGossipFrequency = settings.RegisterDurationSetting( + settings.SystemOnly, + "kv.store_gossip.max_frequency", + "the maximum frequency at which a store will gossip its store descriptor", + defaultMaxStoreGossipFrequency, +) + const ( // defaultGossipWhenCapacityDeltaExceedsFraction specifies the fraction from the // last gossiped store capacity values which need be exceeded before the // store will gossip immediately without waiting for the periodic gossip // interval. - defaultGossipWhenCapacityDeltaExceedsFraction = 0.05 + defaultGossipWhenCapacityDeltaExceedsFraction = 0.10 // gossipWhenLeaseCountDeltaExceeds specifies the absolute change from the // last gossiped store capacity lease count which needs to be exceeded @@ -65,6 +84,12 @@ const ( // if a range lease holder experiences a failure causing a missed // gossip update. systemDataGossipInterval = 1 * time.Minute + + // defaultMaxStoreGossipFrequency is the maximum frequency at which a store + // will gossip its descriptor. Note that periodic gossip will still occur at + // the configured period, regardless of whether last gossip was less than + // maxStoreGossipFrequency ago. + defaultMaxStoreGossipFrequency = 2 * time.Second ) var errPeriodicGossipsDisabled = errors.New("periodic gossip is disabled") @@ -200,6 +225,7 @@ func (s *Store) systemGossipUpdate(sysCfg *config.SystemConfig) { type cachedCapacity struct { syncutil.Mutex cached, lastGossiped roachpb.StoreCapacity + lastGossipedTime time.Time } // StoreGossip is responsible for gossiping the store descriptor. It maintains @@ -223,6 +249,8 @@ type StoreGossip struct { // descriptorGetter is used for getting an up to date or cached store // descriptor to gossip. descriptorGetter StoreDescriptorProvider + sv *settings.Values + clock timeutil.TimeSource } // StoreGossipTestingKnobs defines the testing knobs specific to StoreGossip. @@ -249,13 +277,19 @@ type StoreGossipTestingKnobs struct { // store descriptor: both proactively, calling Gossip() and reacively on // capacity/load changes. func NewStoreGossip( - gossiper InfoGossiper, descGetter StoreDescriptorProvider, testingKnobs StoreGossipTestingKnobs, + gossiper InfoGossiper, + descGetter StoreDescriptorProvider, + testingKnobs StoreGossipTestingKnobs, + sv *settings.Values, + clock timeutil.TimeSource, ) *StoreGossip { return &StoreGossip{ cachedCapacity: &cachedCapacity{}, gossiper: gossiper, descriptorGetter: descGetter, knobs: testingKnobs, + sv: sv, + clock: clock, } } @@ -294,6 +328,20 @@ type InfoGossiper interface { var _ InfoGossiper = &gossip.Gossip{} +// canEagerlyGossipNow checks whether the last gossip was recent enough that we +// should not gossip again now. If the last gossip was too recent, canGossip is +// false, otherwise true. +func (s *StoreGossip) canEagerlyGossipNow() (canGossip bool) { + now := s.clock.Now() + s.cachedCapacity.Lock() + defer s.cachedCapacity.Unlock() + + nextValidGossipTime := s.cachedCapacity.lastGossipedTime.Add( + MaxStoreGossipFrequency.Get(s.sv)) + + return nextValidGossipTime.Before(now) +} + // asyncGossipStore asynchronously gossips the store descriptor, for a given // reason. A cached descriptor is used if specified, otherwise the store // descriptor is updated and capacities recalculated. @@ -340,11 +388,13 @@ func (s *StoreGossip) GossipStore(ctx context.Context, useCached bool) error { // bandwidth and racing with local storepool estimations. // TODO(kvoli): Reconsider what triggers gossip here and possibly limit to // only significant workload changes (load), rather than lease or range - // count. Previoulsy, this was not as much as an issue as the gossip + // count. Previously, this was not as much as an issue as the gossip // interval was 60 seconds, such that gossiping semi-frequently on changes // was required. + now := s.clock.Now() s.cachedCapacity.Lock() s.cachedCapacity.lastGossiped = storeDesc.Capacity + s.cachedCapacity.lastGossipedTime = now s.cachedCapacity.Unlock() // Unique gossip key per store. @@ -421,12 +471,14 @@ func (s *StoreGossip) RecordNewPerSecondStats(newQPS, newWPS float64) { // both absolute and relative terms in order to trigger gossip. func (s *StoreGossip) shouldGossipOnCapacityDelta() (should bool, reason string) { // If there is an ongoing gossip attempt, then there is no need to regossip - // immediately as we will already be gossiping an up to date (cached) capacity. - if s.gossipOngoing.Get() { + // immediately as we will already be gossiping an up to date (cached) + // capacity. If we have recently gossiped the store descriptor, avoid + // re-gossiping too soon, to avoid overloading the receivers of store gossip. + if s.gossipOngoing.Get() || !s.canEagerlyGossipNow() { return } - gossipWhenCapacityDeltaExceedsFraction := defaultGossipWhenCapacityDeltaExceedsFraction + gossipWhenCapacityDeltaExceedsFraction := GossipWhenCapacityDeltaExceedsFraction.Get(s.sv) if overrideCapacityDeltaFraction := s.knobs.OverrideGossipWhenCapacityDeltaExceedsFraction; overrideCapacityDeltaFraction > 0 { gossipWhenCapacityDeltaExceedsFraction = overrideCapacityDeltaFraction } diff --git a/pkg/kv/kvserver/store_gossip_test.go b/pkg/kv/kvserver/store_gossip_test.go index d8a026d5aca3..38bc95e1db98 100644 --- a/pkg/kv/kvserver/store_gossip_test.go +++ b/pkg/kv/kvserver/store_gossip_test.go @@ -11,10 +11,14 @@ package kvserver import ( + math "math" "testing" + "time" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/stretchr/testify/require" ) @@ -28,6 +32,7 @@ func TestStoreGossipDeltaTrigger(t *testing.T) { cached, lastGossiped roachpb.StoreCapacity expectedReason string expectedShould bool + lastGossipTime time.Time }{ { desc: "no delta (empty): shouldn't gossip", @@ -64,15 +69,33 @@ func TestStoreGossipDeltaTrigger(t *testing.T) { expectedReason: "queries-per-second(100.0) writes-per-second(-100.0) range-count(5.0) lease-count(-5.0) change", expectedShould: true, }, + { + desc: "should gossip on all delta but last gossip was too recent", + lastGossiped: roachpb.StoreCapacity{QueriesPerSecond: 100, WritesPerSecond: 100, RangeCount: 10, LeaseCount: 10}, + cached: roachpb.StoreCapacity{QueriesPerSecond: 200, WritesPerSecond: 0, RangeCount: 15, LeaseCount: 5}, + expectedReason: "", + expectedShould: false, + // Set the last gossip time to be some time far in the future, so the + // next gossip time is also far in the future. + lastGossipTime: timeutil.Unix(math.MaxInt64/2, 0), + }, } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { cfg := &StoreConfig{} cfg.SetDefaults(1 /* numStores */) - sg := NewStoreGossip(nil, nil, cfg.TestingKnobs.GossipTestingKnobs) + sg := NewStoreGossip( + nil, + nil, + cfg.TestingKnobs.GossipTestingKnobs, + &cluster.MakeTestingClusterSettings().SV, + timeutil.DefaultTimeSource{}, + ) sg.cachedCapacity.cached = tc.cached sg.cachedCapacity.lastGossiped = tc.lastGossiped + t.Logf("lastGossipTime: %v", tc.lastGossipTime) + sg.cachedCapacity.lastGossipedTime = tc.lastGossipTime should, reason := sg.shouldGossipOnCapacityDelta() require.Equal(t, tc.expectedReason, reason)