From e246b8d2f5d8eeae0bdbacfaf89a005cb0f2e705 Mon Sep 17 00:00:00 2001 From: Pavel Kalinnikov Date: Wed, 24 Jan 2024 09:21:33 +0000 Subject: [PATCH] tracker: add commit index to debug output Signed-off-by: Pavel Kalinnikov --- confchange/testdata/joint_autoleave.txt | 14 ++++---- confchange/testdata/joint_idempotency.txt | 14 ++++---- confchange/testdata/joint_learners_next.txt | 10 +++--- confchange/testdata/joint_safety.txt | 28 +++++++-------- confchange/testdata/simple_idempotency.txt | 30 ++++++++-------- confchange/testdata/simple_promote_demote.txt | 36 +++++++++---------- confchange/testdata/simple_safety.txt | 20 +++++------ confchange/testdata/update.txt | 10 +++--- confchange/testdata/zero.txt | 2 +- testdata/campaign_learner_must_vote.txt | 2 +- testdata/confchange_v1_add_single.txt | 8 ++--- testdata/confchange_v2_add_double_auto.txt | 16 ++++----- .../confchange_v2_add_double_implicit.txt | 8 ++--- testdata/confchange_v2_add_single_auto.txt | 8 ++--- .../confchange_v2_add_single_explicit.txt | 8 ++--- .../heartbeat_resp_recovers_from_probing.txt | 20 +++++------ testdata/lagging_commit.txt | 6 ++-- testdata/replicate_pause.txt | 30 ++++++++-------- testdata/slow_follower_after_compaction.txt | 18 +++++----- testdata/snapshot_succeed_via_app_resp.txt | 24 ++++++------- .../snapshot_succeed_via_app_resp_behind.txt | 26 +++++++------- tracker/progress.go | 2 +- tracker/progress_test.go | 7 ++-- 23 files changed, 174 insertions(+), 173 deletions(-) diff --git a/confchange/testdata/joint_autoleave.txt b/confchange/testdata/joint_autoleave.txt index be138df3..a1370986 100644 --- a/confchange/testdata/joint_autoleave.txt +++ b/confchange/testdata/joint_autoleave.txt @@ -5,16 +5,16 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 # Autoleave is reflected in the config. enter-joint autoleave=true v2 v3 ---- voters=(1 2 3)&&(1) autoleave -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 -3: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 +3: StateProbe match=0 commit=0 next=1 # Can't enter-joint twice, even if autoleave changes. enter-joint autoleave=false @@ -24,6 +24,6 @@ config is already joint leave-joint ---- voters=(1 2 3) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 -3: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 +3: StateProbe match=0 commit=0 next=1 diff --git a/confchange/testdata/joint_idempotency.txt b/confchange/testdata/joint_idempotency.txt index a47f3a66..4f329982 100644 --- a/confchange/testdata/joint_idempotency.txt +++ b/confchange/testdata/joint_idempotency.txt @@ -5,19 +5,19 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 enter-joint r1 r2 r9 v2 v3 v4 v2 v3 v4 l2 l2 r4 r4 l1 l1 ---- voters=(3)&&(1) learners=(2) learners_next=(1) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 learner -3: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 learner +3: StateProbe match=0 commit=0 next=1 leave-joint ---- voters=(3) learners=(1 2) -1: StateProbe match=0 next=1 learner -2: StateProbe match=0 next=1 learner -3: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 learner +2: StateProbe match=0 commit=0 next=1 learner +3: StateProbe match=0 commit=0 next=1 diff --git a/confchange/testdata/joint_learners_next.txt b/confchange/testdata/joint_learners_next.txt index 6faddfe7..81a4bef8 100644 --- a/confchange/testdata/joint_learners_next.txt +++ b/confchange/testdata/joint_learners_next.txt @@ -8,17 +8,17 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 enter-joint v2 l1 ---- voters=(2)&&(1) learners_next=(1) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 leave-joint ---- voters=(2) learners=(1) -1: StateProbe match=0 next=1 learner -2: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 learner +2: StateProbe match=0 commit=0 next=1 diff --git a/confchange/testdata/joint_safety.txt b/confchange/testdata/joint_safety.txt index 75d11b19..7c418a8e 100644 --- a/confchange/testdata/joint_safety.txt +++ b/confchange/testdata/joint_safety.txt @@ -15,7 +15,7 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=3 +1: StateProbe match=0 commit=0 next=3 leave-joint ---- @@ -25,7 +25,7 @@ can't leave a non-joint config enter-joint ---- voters=(1)&&(1) -1: StateProbe match=0 next=3 +1: StateProbe match=0 commit=0 next=3 enter-joint ---- @@ -34,7 +34,7 @@ config is already joint leave-joint ---- voters=(1) -1: StateProbe match=0 next=3 +1: StateProbe match=0 commit=0 next=3 leave-joint ---- @@ -45,10 +45,10 @@ enter-joint r1 v2 v3 l4 ---- voters=(2 3)&&(1) learners=(4) -1: StateProbe match=0 next=3 -2: StateProbe match=0 next=9 -3: StateProbe match=0 next=9 -4: StateProbe match=0 next=9 learner +1: StateProbe match=0 commit=0 next=3 +2: StateProbe match=0 commit=0 next=9 +3: StateProbe match=0 commit=0 next=9 +4: StateProbe match=0 commit=0 next=9 learner enter-joint ---- @@ -67,15 +67,15 @@ can't apply simple config change in joint config leave-joint ---- voters=(2 3) learners=(4) -2: StateProbe match=0 next=9 -3: StateProbe match=0 next=9 -4: StateProbe match=0 next=9 learner +2: StateProbe match=0 commit=0 next=9 +3: StateProbe match=0 commit=0 next=9 +4: StateProbe match=0 commit=0 next=9 learner simple l9 ---- voters=(2 3) learners=(4 9) -2: StateProbe match=0 next=9 -3: StateProbe match=0 next=9 -4: StateProbe match=0 next=9 learner -9: StateProbe match=0 next=14 learner +2: StateProbe match=0 commit=0 next=9 +3: StateProbe match=0 commit=0 next=9 +4: StateProbe match=0 commit=0 next=9 learner +9: StateProbe match=0 commit=0 next=14 learner diff --git a/confchange/testdata/simple_idempotency.txt b/confchange/testdata/simple_idempotency.txt index e31c43b7..0eb7e2ba 100644 --- a/confchange/testdata/simple_idempotency.txt +++ b/confchange/testdata/simple_idempotency.txt @@ -2,68 +2,68 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 simple v2 ---- voters=(1 2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=2 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=2 simple l1 ---- voters=(2) learners=(1) -1: StateProbe match=0 next=1 learner -2: StateProbe match=0 next=2 +1: StateProbe match=0 commit=0 next=1 learner +2: StateProbe match=0 commit=0 next=2 simple l1 ---- voters=(2) learners=(1) -1: StateProbe match=0 next=1 learner -2: StateProbe match=0 next=2 +1: StateProbe match=0 commit=0 next=1 learner +2: StateProbe match=0 commit=0 next=2 simple r1 ---- voters=(2) -2: StateProbe match=0 next=2 +2: StateProbe match=0 commit=0 next=2 simple r1 ---- voters=(2) -2: StateProbe match=0 next=2 +2: StateProbe match=0 commit=0 next=2 simple v3 ---- voters=(2 3) -2: StateProbe match=0 next=2 -3: StateProbe match=0 next=7 +2: StateProbe match=0 commit=0 next=2 +3: StateProbe match=0 commit=0 next=7 simple r3 ---- voters=(2) -2: StateProbe match=0 next=2 +2: StateProbe match=0 commit=0 next=2 simple r3 ---- voters=(2) -2: StateProbe match=0 next=2 +2: StateProbe match=0 commit=0 next=2 simple r4 ---- voters=(2) -2: StateProbe match=0 next=2 +2: StateProbe match=0 commit=0 next=2 diff --git a/confchange/testdata/simple_promote_demote.txt b/confchange/testdata/simple_promote_demote.txt index b4b770de..5e1eaefc 100644 --- a/confchange/testdata/simple_promote_demote.txt +++ b/confchange/testdata/simple_promote_demote.txt @@ -4,22 +4,22 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 simple v2 ---- voters=(1 2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 simple v3 ---- voters=(1 2 3) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 -3: StateProbe match=0 next=2 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 +3: StateProbe match=0 commit=0 next=2 # Can atomically demote and promote without a hitch. # This is pointless, but possible. @@ -27,18 +27,18 @@ simple l1 v1 ---- voters=(1 2 3) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 -3: StateProbe match=0 next=2 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 +3: StateProbe match=0 commit=0 next=2 # Can demote a voter. simple l2 ---- voters=(1 3) learners=(2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 learner -3: StateProbe match=0 next=2 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 learner +3: StateProbe match=0 commit=0 next=2 # Can atomically promote and demote the same voter. # This is pointless, but possible. @@ -46,15 +46,15 @@ simple v2 l2 ---- voters=(1 3) learners=(2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 learner -3: StateProbe match=0 next=2 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 learner +3: StateProbe match=0 commit=0 next=2 # Can promote a voter. simple v2 ---- voters=(1 2 3) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 -3: StateProbe match=0 next=2 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 +3: StateProbe match=0 commit=0 next=2 diff --git a/confchange/testdata/simple_safety.txt b/confchange/testdata/simple_safety.txt index 6566c5fc..42f30350 100644 --- a/confchange/testdata/simple_safety.txt +++ b/confchange/testdata/simple_safety.txt @@ -7,15 +7,15 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 simple v2 l3 ---- voters=(1 2) learners=(3) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=2 -3: StateProbe match=0 next=2 learner +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=2 +3: StateProbe match=0 commit=0 next=2 learner simple r1 v5 @@ -46,11 +46,11 @@ simple l2 l3 l4 l5 ---- voters=(1) learners=(2 3 4 5) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=2 learner -3: StateProbe match=0 next=2 learner -4: StateProbe match=0 next=8 learner -5: StateProbe match=0 next=8 learner +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=2 learner +3: StateProbe match=0 commit=0 next=2 learner +4: StateProbe match=0 commit=0 next=8 learner +5: StateProbe match=0 commit=0 next=8 learner simple r1 @@ -61,4 +61,4 @@ simple r2 r3 r4 r5 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 diff --git a/confchange/testdata/update.txt b/confchange/testdata/update.txt index ac47bf3e..1220480e 100644 --- a/confchange/testdata/update.txt +++ b/confchange/testdata/update.txt @@ -6,18 +6,18 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 simple v2 u1 ---- voters=(1 2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 simple u1 u2 u3 u1 u2 u3 ---- voters=(1 2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 +2: StateProbe match=0 commit=0 next=1 diff --git a/confchange/testdata/zero.txt b/confchange/testdata/zero.txt index 226ade08..068e5bec 100644 --- a/confchange/testdata/zero.txt +++ b/confchange/testdata/zero.txt @@ -3,4 +3,4 @@ simple v1 r0 v0 l0 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 commit=0 next=1 diff --git a/testdata/campaign_learner_must_vote.txt b/testdata/campaign_learner_must_vote.txt index 9da576ff..f4d62888 100644 --- a/testdata/campaign_learner_must_vote.txt +++ b/testdata/campaign_learner_must_vote.txt @@ -113,7 +113,7 @@ stabilize 2 3 > 2 receiving messages 3->2 MsgAppResp Term:2 Log:1/4 Rejected (Hint: 3) DEBUG 2 received MsgAppResp(rejected, hint: (index 3, term 1)) from 3 for index 4 - DEBUG 2 decreased progress of 3 to [StateProbe match=0 next=4] + DEBUG 2 decreased progress of 3 to [StateProbe match=0 commit=0 next=4] > 2 handling Ready Ready MustSync=false: Messages: diff --git a/testdata/confchange_v1_add_single.txt b/testdata/confchange_v1_add_single.txt index f13d3077..6d300c23 100644 --- a/testdata/confchange_v1_add_single.txt +++ b/testdata/confchange_v1_add_single.txt @@ -70,9 +70,9 @@ stabilize > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 - DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] + DEBUG 1 decreased progress of 2 to [StateProbe match=0 commit=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 commit=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 commit=0 next=5 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -93,4 +93,4 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/4 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 commit=0 next=5 paused pendingSnap=4] diff --git a/testdata/confchange_v2_add_double_auto.txt b/testdata/confchange_v2_add_double_auto.txt index 0d86ac1a..1adf9436 100644 --- a/testdata/confchange_v2_add_double_auto.txt +++ b/testdata/confchange_v2_add_double_auto.txt @@ -93,9 +93,9 @@ stabilize 1 2 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 - DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] + DEBUG 1 decreased progress of 2 to [StateProbe match=0 commit=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 commit=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 commit=0 next=5 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -116,7 +116,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/4 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 commit=0 next=5 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -169,9 +169,9 @@ stabilize 1 3 > 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 3 for index 3 - DEBUG 1 decreased progress of 3 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 5] sent snapshot[index: 5, term: 1] to 3 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=6 paused pendingSnap=5] + DEBUG 1 decreased progress of 3 to [StateProbe match=0 commit=0 next=1] + DEBUG 1 [firstindex: 3, commit: 5] sent snapshot[index: 5, term: 1] to 3 [StateProbe match=0 commit=0 next=1] + DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 commit=0 next=6 paused pendingSnap=5] > 1 handling Ready Ready MustSync=false: Messages: @@ -192,7 +192,7 @@ stabilize 1 3 3->1 MsgAppResp Term:1 Log:0/5 > 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/5 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=5 next=6 paused pendingSnap=5] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=5 commit=0 next=6 paused pendingSnap=5] # Nothing else happens. stabilize diff --git a/testdata/confchange_v2_add_double_implicit.txt b/testdata/confchange_v2_add_double_implicit.txt index 30f26f68..3c7ffb8e 100644 --- a/testdata/confchange_v2_add_double_implicit.txt +++ b/testdata/confchange_v2_add_double_implicit.txt @@ -76,9 +76,9 @@ stabilize 1 2 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 - DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] + DEBUG 1 decreased progress of 2 to [StateProbe match=0 commit=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 commit=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 commit=0 next=5 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -99,7 +99,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/4 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 commit=0 next=5 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: diff --git a/testdata/confchange_v2_add_single_auto.txt b/testdata/confchange_v2_add_single_auto.txt index 3c234159..489b6485 100644 --- a/testdata/confchange_v2_add_single_auto.txt +++ b/testdata/confchange_v2_add_single_auto.txt @@ -71,9 +71,9 @@ stabilize > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 - DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] + DEBUG 1 decreased progress of 2 to [StateProbe match=0 commit=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 commit=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 commit=0 next=5 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -94,4 +94,4 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/4 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 commit=0 next=5 paused pendingSnap=4] diff --git a/testdata/confchange_v2_add_single_explicit.txt b/testdata/confchange_v2_add_single_explicit.txt index 770c9a27..2f2420f8 100644 --- a/testdata/confchange_v2_add_single_explicit.txt +++ b/testdata/confchange_v2_add_single_explicit.txt @@ -71,9 +71,9 @@ stabilize 1 2 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 - DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] + DEBUG 1 decreased progress of 2 to [StateProbe match=0 commit=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 commit=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 commit=0 next=5 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -94,7 +94,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/4 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 commit=0 next=5 paused pendingSnap=4] # Check that we're not allowed to change membership again while in the joint state. # This leads to an empty entry being proposed instead (index 5 in the stabilize block diff --git a/testdata/heartbeat_resp_recovers_from_probing.txt b/testdata/heartbeat_resp_recovers_from_probing.txt index bb6eef1e..e94a67a1 100644 --- a/testdata/heartbeat_resp_recovers_from_probing.txt +++ b/testdata/heartbeat_resp_recovers_from_probing.txt @@ -28,20 +28,20 @@ ok status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateReplicate match=11 next=12 +1: StateReplicate match=11 commit=10 next=12 +2: StateReplicate match=11 commit=11 next=12 +3: StateReplicate match=11 commit=11 next=12 # On the first replica, report the second one as not reachable. report-unreachable 1 2 ---- -DEBUG 1 failed to send message to 2 because it is unreachable [StateProbe match=11 next=12] +DEBUG 1 failed to send message to 2 because it is unreachable [StateProbe match=11 commit=11 next=12] status 1 ---- -1: StateReplicate match=11 next=12 -2: StateProbe match=11 next=12 -3: StateReplicate match=11 next=12 +1: StateReplicate match=11 commit=10 next=12 +2: StateProbe match=11 commit=11 next=12 +3: StateReplicate match=11 commit=11 next=12 tick-heartbeat 1 ---- @@ -85,6 +85,6 @@ stabilize status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateReplicate match=11 next=12 +1: StateReplicate match=11 commit=10 next=12 +2: StateReplicate match=11 commit=11 next=12 +3: StateReplicate match=11 commit=11 next=12 diff --git a/testdata/lagging_commit.txt b/testdata/lagging_commit.txt index 551d4e75..66880ba4 100644 --- a/testdata/lagging_commit.txt +++ b/testdata/lagging_commit.txt @@ -114,9 +114,9 @@ dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:13 status 1 ---- -1: StateReplicate match=13 next=14 -2: StateReplicate match=13 next=14 -3: StateReplicate match=11 next=14 inflight=2 +1: StateReplicate match=13 commit=11 next=14 +2: StateReplicate match=13 commit=13 next=14 +3: StateReplicate match=11 commit=11 next=14 inflight=2 # The leader still observes that the entries are in-flight to the follower 3, # since it hasn't heard from it. Nothing triggers updating the follower's diff --git a/testdata/replicate_pause.txt b/testdata/replicate_pause.txt index d9cee59f..b9a359bf 100644 --- a/testdata/replicate_pause.txt +++ b/testdata/replicate_pause.txt @@ -46,9 +46,9 @@ ok # Expect that in-flight tracking to nodes 2 and 3 is saturated. status 1 ---- -1: StateReplicate match=14 next=15 -2: StateReplicate match=11 next=15 paused inflight=3[full] -3: StateReplicate match=11 next=15 paused inflight=3[full] +1: StateReplicate match=14 commit=11 next=15 +2: StateReplicate match=11 commit=11 next=15 paused inflight=3[full] +3: StateReplicate match=11 commit=11 next=15 paused inflight=3[full] log-level none ---- @@ -66,9 +66,9 @@ ok # Expect that the entries are committed and stored on nodes 1 and 2. status 1 ---- -1: StateReplicate match=14 next=15 -2: StateReplicate match=14 next=15 -3: StateReplicate match=11 next=15 paused inflight=3[full] +1: StateReplicate match=14 commit=11 next=15 +2: StateReplicate match=14 commit=14 next=15 +3: StateReplicate match=11 commit=11 next=15 paused inflight=3[full] # Drop append messages to node 3. deliver-msgs drop=3 @@ -94,9 +94,9 @@ ok # In-flight tracking to nodes 2 and 3 is saturated, but node 3 is behind. status 1 ---- -1: StateReplicate match=14 next=15 -2: StateReplicate match=14 next=18 paused inflight=3[full] -3: StateReplicate match=11 next=15 paused inflight=3[full] +1: StateReplicate match=14 commit=11 next=15 +2: StateReplicate match=14 commit=14 next=18 paused inflight=3[full] +3: StateReplicate match=11 commit=11 next=15 paused inflight=3[full] log-level none ---- @@ -114,9 +114,9 @@ ok # Expect that the entries are committed and stored only on nodes 1 and 2. status 1 ---- -1: StateReplicate match=17 next=18 -2: StateReplicate match=17 next=18 -3: StateReplicate match=11 next=15 paused inflight=3[full] +1: StateReplicate match=17 commit=14 next=18 +2: StateReplicate match=17 commit=17 next=18 +3: StateReplicate match=11 commit=11 next=15 paused inflight=3[full] # Make a heartbeat roundtrip. tick-heartbeat 1 @@ -185,6 +185,6 @@ ok # Eventually all nodes catch up on the committed state. status 1 ---- -1: StateReplicate match=17 next=18 -2: StateReplicate match=17 next=18 -3: StateReplicate match=17 next=18 +1: StateReplicate match=17 commit=14 next=18 +2: StateReplicate match=17 commit=17 next=18 +3: StateReplicate match=17 commit=17 next=18 diff --git a/testdata/slow_follower_after_compaction.txt b/testdata/slow_follower_after_compaction.txt index 0d3d48c8..7d6ad29f 100644 --- a/testdata/slow_follower_after_compaction.txt +++ b/testdata/slow_follower_after_compaction.txt @@ -43,9 +43,9 @@ ok # All nodes up-to-date. status 1 ---- -1: StateReplicate match=14 next=15 -2: StateReplicate match=14 next=15 -3: StateReplicate match=14 next=15 +1: StateReplicate match=14 commit=11 next=15 +2: StateReplicate match=14 commit=14 next=15 +3: StateReplicate match=14 commit=14 next=15 log-level none ---- @@ -79,9 +79,9 @@ ok # Nodes 1 and 2 up-to-date, 3 is behind and MsgApp flow is throttled. status 1 ---- -1: StateReplicate match=18 next=19 -2: StateReplicate match=18 next=19 -3: StateReplicate match=14 next=17 paused inflight=2[full] +1: StateReplicate match=18 commit=14 next=19 +2: StateReplicate match=18 commit=18 next=19 +3: StateReplicate match=14 commit=14 next=17 paused inflight=2[full] # Break the MsgApp flow from the leader to node 3. deliver-msgs drop=3 @@ -116,6 +116,6 @@ ok # All nodes caught up. status 1 ---- -1: StateReplicate match=18 next=19 -2: StateReplicate match=18 next=19 -3: StateReplicate match=18 next=19 +1: StateReplicate match=18 commit=14 next=19 +2: StateReplicate match=18 commit=18 next=19 +3: StateReplicate match=18 commit=14 next=19 diff --git a/testdata/snapshot_succeed_via_app_resp.txt b/testdata/snapshot_succeed_via_app_resp.txt index b1fc4144..a015f9de 100644 --- a/testdata/snapshot_succeed_via_app_resp.txt +++ b/testdata/snapshot_succeed_via_app_resp.txt @@ -41,9 +41,9 @@ ok status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateProbe match=0 next=11 paused inactive +1: StateReplicate match=11 commit=10 next=12 +2: StateReplicate match=11 commit=11 next=12 +3: StateProbe match=0 commit=0 next=11 paused inactive # Add the node that will receive a snapshot (it has no state at all, does not # even have a config). @@ -86,8 +86,8 @@ stabilize 1 ---- > 1 receiving messages 3->1 MsgHeartbeatResp Term:1 Log:0/0 - DEBUG 1 [firstindex: 12, commit: 11] sent snapshot[index: 11, term: 1] to 3 [StateProbe match=0 next=11] - DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=12 paused pendingSnap=11] + DEBUG 1 [firstindex: 12, commit: 11] sent snapshot[index: 11, term: 1] to 3 [StateProbe match=0 commit=0 next=11] + DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 commit=0 next=12 paused pendingSnap=11] > 1 handling Ready Ready MustSync=false: Messages: @@ -96,9 +96,9 @@ stabilize 1 status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateSnapshot match=0 next=12 paused pendingSnap=11 +1: StateReplicate match=11 commit=10 next=12 +2: StateReplicate match=11 commit=11 next=12 +3: StateSnapshot match=0 commit=0 next=12 paused pendingSnap=11 # Follower applies the snapshot. Note how it reacts with a MsgAppResp upon completion. # The snapshot fully catches the follower up (i.e. there are no more log entries it @@ -126,13 +126,13 @@ stabilize 1 ---- > 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/11 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 next=12 paused pendingSnap=11] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 commit=0 next=12 paused pendingSnap=11] status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateReplicate match=11 next=12 +1: StateReplicate match=11 commit=10 next=12 +2: StateReplicate match=11 commit=11 next=12 +3: StateReplicate match=11 commit=0 next=12 # Let things settle. stabilize diff --git a/testdata/snapshot_succeed_via_app_resp_behind.txt b/testdata/snapshot_succeed_via_app_resp_behind.txt index 3bab832d..b27f19e8 100644 --- a/testdata/snapshot_succeed_via_app_resp_behind.txt +++ b/testdata/snapshot_succeed_via_app_resp_behind.txt @@ -48,9 +48,9 @@ raft-state status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateProbe match=0 next=11 paused inactive +1: StateReplicate match=11 commit=10 next=12 +2: StateReplicate match=11 commit=11 next=12 +3: StateProbe match=0 commit=0 next=11 paused inactive raft-log 3 ---- @@ -83,9 +83,9 @@ ok status 1 ---- -1: StateReplicate match=12 next=13 -2: StateReplicate match=12 next=13 -3: StateProbe match=0 next=11 paused inactive +1: StateReplicate match=12 commit=11 next=13 +2: StateReplicate match=12 commit=12 next=13 +3: StateProbe match=0 commit=0 next=11 paused inactive # 3 now gets the first MsgApp the leader originally sent, trying to append entry # 11 but this is rejected because the follower's log started at index 5. @@ -122,9 +122,9 @@ stabilize 1 > 1 receiving messages 3->1 MsgAppResp Term:1 Log:1/10 Rejected (Hint: 5) DEBUG 1 received MsgAppResp(rejected, hint: (index 5, term 1)) from 3 for index 10 - DEBUG 1 decreased progress of 3 to [StateProbe match=0 next=6] - DEBUG 1 [firstindex: 11, commit: 12] sent snapshot[index: 12, term: 1] to 3 [StateProbe match=0 next=6] - DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=13 paused pendingSnap=12] + DEBUG 1 decreased progress of 3 to [StateProbe match=0 commit=0 next=6] + DEBUG 1 [firstindex: 11, commit: 12] sent snapshot[index: 12, term: 1] to 3 [StateProbe match=0 commit=0 next=6] + DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 commit=0 next=13 paused pendingSnap=12] > 1 handling Ready Ready MustSync=false: Messages: @@ -152,7 +152,7 @@ stabilize 1 ---- > 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/11 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 next=13 paused pendingSnap=12] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 commit=0 next=13 paused pendingSnap=12] > 1 handling Ready Ready MustSync=false: Messages: @@ -162,6 +162,6 @@ stabilize 1 # This is despite its PendingSnapshot having been 12. status 1 ---- -1: StateReplicate match=12 next=13 -2: StateReplicate match=12 next=13 -3: StateReplicate match=11 next=13 inflight=1 +1: StateReplicate match=12 commit=11 next=13 +2: StateReplicate match=12 commit=12 next=13 +3: StateReplicate match=11 commit=0 next=13 inflight=1 diff --git a/tracker/progress.go b/tracker/progress.go index 905dfedc..ebf52fb0 100644 --- a/tracker/progress.go +++ b/tracker/progress.go @@ -286,7 +286,7 @@ func (pr *Progress) IsPaused() bool { func (pr *Progress) String() string { var buf strings.Builder - fmt.Fprintf(&buf, "%s match=%d next=%d", pr.State, pr.Match, pr.Next) + fmt.Fprintf(&buf, "%s match=%d commit=%d next=%d", pr.State, pr.Match, pr.Commit, pr.Next) if pr.IsLearner { fmt.Fprint(&buf, " learner") } diff --git a/tracker/progress_test.go b/tracker/progress_test.go index 49dedb53..702aef0d 100644 --- a/tracker/progress_test.go +++ b/tracker/progress_test.go @@ -24,8 +24,9 @@ func TestProgressString(t *testing.T) { ins := NewInflights(1, 0) ins.Add(123, 1) pr := &Progress{ - Match: 1, - Next: 2, + Match: 2, + Commit: 1, + Next: 3, State: StateSnapshot, PendingSnapshot: 123, RecentActive: false, @@ -33,7 +34,7 @@ func TestProgressString(t *testing.T) { IsLearner: true, Inflights: ins, } - const exp = `StateSnapshot match=1 next=2 learner paused pendingSnap=123 inactive inflight=1[full]` + const exp = `StateSnapshot match=2 commit=1 next=3 learner paused pendingSnap=123 inactive inflight=1[full]` assert.Equal(t, exp, pr.String()) }