From da6ad00455b36686a60003784ac617b36edbed3a Mon Sep 17 00:00:00 2001 From: Pavel Kalinnikov Date: Wed, 24 Jan 2024 09:20:01 +0000 Subject: [PATCH] raft,tracker: track commit index of each follower Signed-off-by: Pavel Kalinnikov --- raft.go | 11 +- testdata/async_storage_writes.txt | 142 +++++++++--------- .../async_storage_writes_append_aba_race.txt | 18 +-- testdata/campaign.txt | 16 +- testdata/campaign_learner_must_vote.txt | 8 +- testdata/checkquorum.txt | 16 +- testdata/confchange_v1_add_single.txt | 4 +- testdata/confchange_v1_remove_leader.txt | 48 +++--- .../confchange_v1_remove_leader_stepdown.txt | 40 ++--- testdata/confchange_v2_add_double_auto.txt | 76 +++++----- .../confchange_v2_add_double_implicit.txt | 8 +- testdata/confchange_v2_add_single_auto.txt | 4 +- .../confchange_v2_add_single_explicit.txt | 28 ++-- testdata/confchange_v2_replace_leader.txt | 72 ++++----- .../confchange_v2_replace_leader_stepdown.txt | 24 +-- .../forget_leader_prevote_checkquorum.txt | 2 +- .../heartbeat_resp_recovers_from_probing.txt | 4 +- testdata/prevote.txt | 32 ++-- testdata/prevote_checkquorum.txt | 32 ++-- testdata/probe_and_replicate.txt | 48 +++--- testdata/snapshot_succeed_via_app_resp.txt | 4 +- tracker/progress.go | 11 ++ 22 files changed, 331 insertions(+), 317 deletions(-) diff --git a/raft.go b/raft.go index 5a150562..8c76573d 100644 --- a/raft.go +++ b/raft.go @@ -815,7 +815,7 @@ func (r *raft) appendEntry(es ...pb.Entry) (accepted bool) { // if r.maybeCommit() { // r.bcastAppend() // } - r.send(pb.Message{To: r.id, Type: pb.MsgAppResp, Index: li}) + r.send(pb.Message{To: r.id, Type: pb.MsgAppResp, Index: li, Commit: r.raftLog.committed}) return true } @@ -1465,6 +1465,7 @@ func stepLeader(r *raft, m pb.Message) error { r.sendAppend(m.From) } } else { + pr.UpdateCommit(m.Commit) oldPaused := pr.IsPaused() // We want to update our tracking if the response updates our // matched index or if the response can move a probing peer back @@ -1505,6 +1506,8 @@ func stepLeader(r *raft, m pb.Message) error { } else if oldPaused { // If we were paused before, this node may be missing the // latest commit index, so send it. + // TODO(pav-kv): remove this branch, and decide on sending the commit + // index update based on pr.Commit. r.sendAppend(m.From) } // We've updated flow control information above, which may @@ -1730,12 +1733,12 @@ func stepFollower(r *raft, m pb.Message) error { } func (r *raft) handleAppendEntries(m pb.Message) { - if m.Index < r.raftLog.committed { - r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: r.raftLog.committed}) + if committed := r.raftLog.committed; m.Index < committed { + r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: committed, Commit: committed}) return } if mlastIndex, ok := r.raftLog.maybeAppend(m.Index, m.LogTerm, m.Commit, m.Entries...); ok { - r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: mlastIndex}) + r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: mlastIndex, Commit: m.Commit}) return } r.logger.Debugf("%x [logterm: %d, index: %d] rejected MsgApp [logterm: %d, index: %d] from %x", diff --git a/testdata/async_storage_writes.txt b/testdata/async_storage_writes.txt index 52f37092..13fb015f 100644 --- a/testdata/async_storage_writes.txt +++ b/testdata/async_storage_writes.txt @@ -83,7 +83,7 @@ stabilize Messages: 1->2 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] 1->3 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] - 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[1->1 MsgAppResp Term:1 Log:0/11, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[1->1 MsgAppResp Term:1 Log:0/11 Commit:10, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11] > 2 receiving messages 1->2 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] > 3 receiving messages @@ -92,7 +92,7 @@ stabilize Processing: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses: - 1->1 MsgAppResp Term:1 Log:0/11 + 1->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 > 2 handling Ready Ready MustSync=true: @@ -100,32 +100,32 @@ stabilize Entries: 1/11 EntryNormal "" Messages: - 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[2->1 MsgAppResp Term:1 Log:0/11, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11] + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[2->1 MsgAppResp Term:1 Log:0/11 Commit:10, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11] > 3 handling Ready Ready MustSync=true: Lead:1 State:StateFollower Entries: 1/11 EntryNormal "" Messages: - 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[3->1 MsgAppResp Term:1 Log:0/11, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11] + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[3->1 MsgAppResp Term:1 Log:0/11 Commit:10, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11] > 1 receiving messages - 1->1 MsgAppResp Term:1 Log:0/11 + 1->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses: - 2->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses: - 3->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/11 - 3->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:10 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:10 > 2 receiving messages AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 > 3 receiving messages @@ -159,7 +159,7 @@ stabilize CommittedEntries: 1/11 EntryNormal "" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses:[2->1 MsgAppResp Term:1 Log:0/11] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses:[2->1 MsgAppResp Term:1 Log:0/11 Commit:11] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""]] > 3 handling Ready Ready MustSync=false: @@ -167,7 +167,7 @@ stabilize CommittedEntries: 1/11 EntryNormal "" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/11] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/11 Commit:11] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""]] > 1 receiving messages ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] @@ -175,12 +175,12 @@ stabilize Processing: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses: - 2->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:11 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses: - 3->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:11 > 2 processing apply thread Processing: 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] @@ -192,8 +192,8 @@ stabilize Responses: ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/11 - 3->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:11 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:11 > 2 receiving messages ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] > 3 receiving messages @@ -212,7 +212,7 @@ process-ready 1 2 3 Messages: 1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] 1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] - 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[1->1 MsgAppResp Term:1 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[1->1 MsgAppResp Term:1 Log:0/12 Commit:11, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12] > 2 handling Ready > 3 handling Ready @@ -232,13 +232,13 @@ process-ready 1 2 3 Entries: 1/12 EntryNormal "prop_1" Messages: - 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[2->1 MsgAppResp Term:1 Log:0/12, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12] + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[2->1 MsgAppResp Term:1 Log:0/12 Commit:11, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12] > 3 handling Ready Ready MustSync=true: Entries: 1/12 EntryNormal "prop_1" Messages: - 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[3->1 MsgAppResp Term:1 Log:0/12, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12] + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[3->1 MsgAppResp Term:1 Log:0/12 Commit:11, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12] propose 1 prop_2 ---- @@ -253,7 +253,7 @@ process-ready 1 2 3 Messages: 1->2 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_2"] 1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_2"] - 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[1->1 MsgAppResp Term:1 Log:0/13, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[1->1 MsgAppResp Term:1 Log:0/13 Commit:11, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13] > 2 handling Ready > 3 handling Ready @@ -273,13 +273,13 @@ process-ready 1 2 3 Entries: 1/13 EntryNormal "prop_2" Messages: - 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[2->1 MsgAppResp Term:1 Log:0/13, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13] + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[2->1 MsgAppResp Term:1 Log:0/13 Commit:11, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13] > 3 handling Ready Ready MustSync=true: Entries: 1/13 EntryNormal "prop_2" Messages: - 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[3->1 MsgAppResp Term:1 Log:0/13, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13] + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[3->1 MsgAppResp Term:1 Log:0/13 Commit:11, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13] process-append-thread 1 2 3 ---- @@ -287,27 +287,27 @@ process-append-thread 1 2 3 Processing: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses: - 1->1 MsgAppResp Term:1 Log:0/12 + 1->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses: - 2->1 MsgAppResp Term:1 Log:0/12 + 2->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses: - 3->1 MsgAppResp Term:1 Log:0/12 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12 deliver-msgs 1 2 3 ---- -1->1 MsgAppResp Term:1 Log:0/12 +1->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 -2->1 MsgAppResp Term:1 Log:0/12 -3->1 MsgAppResp Term:1 Log:0/12 +2->1 MsgAppResp Term:1 Log:0/12 Commit:11 +3->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12 @@ -329,7 +329,7 @@ process-ready 1 2 3 1->3 MsgApp Term:1 Log:1/13 Commit:12 1->2 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"] 1->3 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"] - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[1->1 MsgAppResp Term:1 Log:0/14, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14] + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[1->1 MsgAppResp Term:1 Log:0/14 Commit:12, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14] 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"]] > 2 handling Ready @@ -355,7 +355,7 @@ process-ready 1 2 3 CommittedEntries: 1/12 EntryNormal "prop_1" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[2->1 MsgAppResp Term:1 Log:0/13, 2->1 MsgAppResp Term:1 Log:0/14, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[2->1 MsgAppResp Term:1 Log:0/13 Commit:12, 2->1 MsgAppResp Term:1 Log:0/14 Commit:12, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"]] > 3 handling Ready Ready MustSync=true: @@ -365,7 +365,7 @@ process-ready 1 2 3 CommittedEntries: 1/12 EntryNormal "prop_1" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[3->1 MsgAppResp Term:1 Log:0/13, 3->1 MsgAppResp Term:1 Log:0/14, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[3->1 MsgAppResp Term:1 Log:0/13 Commit:12, 3->1 MsgAppResp Term:1 Log:0/14 Commit:12, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"]] process-append-thread 1 2 3 @@ -374,27 +374,27 @@ process-append-thread 1 2 3 Processing: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses: - 1->1 MsgAppResp Term:1 Log:0/13 + 1->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13 > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses: - 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses: - 3->1 MsgAppResp Term:1 Log:0/13 + 3->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13 deliver-msgs 1 2 3 ---- -1->1 MsgAppResp Term:1 Log:0/13 +1->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13 -2->1 MsgAppResp Term:1 Log:0/13 -3->1 MsgAppResp Term:1 Log:0/13 +2->1 MsgAppResp Term:1 Log:0/13 Commit:11 +3->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13 @@ -416,7 +416,7 @@ process-ready 1 2 3 1->3 MsgApp Term:1 Log:1/14 Commit:13 1->2 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"] 1->3 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"] - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[1->1 MsgAppResp Term:1 Log:0/15, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15] + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[1->1 MsgAppResp Term:1 Log:0/15 Commit:13, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15] 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"]] > 2 handling Ready @@ -442,7 +442,7 @@ process-ready 1 2 3 CommittedEntries: 1/13 EntryNormal "prop_2" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[2->1 MsgAppResp Term:1 Log:0/14, 2->1 MsgAppResp Term:1 Log:0/15, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[2->1 MsgAppResp Term:1 Log:0/14 Commit:13, 2->1 MsgAppResp Term:1 Log:0/15 Commit:13, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"]] > 3 handling Ready Ready MustSync=true: @@ -452,7 +452,7 @@ process-ready 1 2 3 CommittedEntries: 1/13 EntryNormal "prop_2" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[3->1 MsgAppResp Term:1 Log:0/14, 3->1 MsgAppResp Term:1 Log:0/15, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[3->1 MsgAppResp Term:1 Log:0/14 Commit:13, 3->1 MsgAppResp Term:1 Log:0/15 Commit:13, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"]] process-append-thread 1 2 3 @@ -461,21 +461,21 @@ process-append-thread 1 2 3 Processing: 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses: - 1->1 MsgAppResp Term:1 Log:0/14 + 1->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14 > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses: - 2->1 MsgAppResp Term:1 Log:0/13 - 2->1 MsgAppResp Term:1 Log:0/14 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:12 + 2->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses: - 3->1 MsgAppResp Term:1 Log:0/13 - 3->1 MsgAppResp Term:1 Log:0/14 + 3->1 MsgAppResp Term:1 Log:0/13 Commit:12 + 3->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14 process-apply-thread 1 2 3 @@ -498,12 +498,12 @@ process-apply-thread 1 2 3 deliver-msgs 1 2 3 ---- -1->1 MsgAppResp Term:1 Log:0/14 +1->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14 -2->1 MsgAppResp Term:1 Log:0/13 -2->1 MsgAppResp Term:1 Log:0/14 -3->1 MsgAppResp Term:1 Log:0/13 -3->1 MsgAppResp Term:1 Log:0/14 +2->1 MsgAppResp Term:1 Log:0/13 Commit:12 +2->1 MsgAppResp Term:1 Log:0/14 Commit:12 +3->1 MsgAppResp Term:1 Log:0/13 Commit:12 +3->1 MsgAppResp Term:1 Log:0/14 Commit:12 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14 ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] @@ -542,7 +542,7 @@ process-ready 1 2 3 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[2->1 MsgAppResp Term:1 Log:0/15, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[2->1 MsgAppResp Term:1 Log:0/15 Commit:14, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]] > 3 handling Ready Ready MustSync=false: @@ -550,7 +550,7 @@ process-ready 1 2 3 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15 Commit:14, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]] process-append-thread 1 2 3 @@ -559,21 +559,21 @@ process-append-thread 1 2 3 Processing: 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses: - 1->1 MsgAppResp Term:1 Log:0/15 + 1->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses: - 2->1 MsgAppResp Term:1 Log:0/14 - 2->1 MsgAppResp Term:1 Log:0/15 + 2->1 MsgAppResp Term:1 Log:0/14 Commit:13 + 2->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses: - 3->1 MsgAppResp Term:1 Log:0/14 - 3->1 MsgAppResp Term:1 Log:0/15 + 3->1 MsgAppResp Term:1 Log:0/14 Commit:13 + 3->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 process-apply-thread 1 2 3 @@ -596,12 +596,12 @@ process-apply-thread 1 2 3 deliver-msgs 1 2 3 ---- -1->1 MsgAppResp Term:1 Log:0/15 +1->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 -2->1 MsgAppResp Term:1 Log:0/14 -2->1 MsgAppResp Term:1 Log:0/15 -3->1 MsgAppResp Term:1 Log:0/14 -3->1 MsgAppResp Term:1 Log:0/15 +2->1 MsgAppResp Term:1 Log:0/14 Commit:13 +2->1 MsgAppResp Term:1 Log:0/15 Commit:13 +3->1 MsgAppResp Term:1 Log:0/14 Commit:13 +3->1 MsgAppResp Term:1 Log:0/15 Commit:13 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] @@ -640,7 +640,7 @@ process-ready 1 2 3 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[2->1 MsgAppResp Term:1 Log:0/15] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[2->1 MsgAppResp Term:1 Log:0/15 Commit:15] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]] > 3 handling Ready Ready MustSync=false: @@ -648,7 +648,7 @@ process-ready 1 2 3 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15 Commit:15] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]] process-append-thread 2 3 @@ -657,13 +657,13 @@ process-append-thread 2 3 Processing: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses: - 2->1 MsgAppResp Term:1 Log:0/15 + 2->1 MsgAppResp Term:1 Log:0/15 Commit:14 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses: - 3->1 MsgAppResp Term:1 Log:0/15 + 3->1 MsgAppResp Term:1 Log:0/15 Commit:14 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 process-apply-thread 1 2 3 @@ -686,8 +686,8 @@ process-apply-thread 1 2 3 deliver-msgs 1 2 3 ---- -2->1 MsgAppResp Term:1 Log:0/15 -3->1 MsgAppResp Term:1 Log:0/15 +2->1 MsgAppResp Term:1 Log:0/15 Commit:14 +3->1 MsgAppResp Term:1 Log:0/15 Commit:14 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 INFO entry at index 15 missing from unstable log; ignoring @@ -711,12 +711,12 @@ process-append-thread 2 3 Processing: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses: - 2->1 MsgAppResp Term:1 Log:0/15 + 2->1 MsgAppResp Term:1 Log:0/15 Commit:15 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses: - 3->1 MsgAppResp Term:1 Log:0/15 + 3->1 MsgAppResp Term:1 Log:0/15 Commit:15 process-apply-thread 1 2 3 ---- @@ -738,8 +738,8 @@ process-apply-thread 1 2 3 deliver-msgs 1 2 3 ---- -2->1 MsgAppResp Term:1 Log:0/15 -3->1 MsgAppResp Term:1 Log:0/15 +2->1 MsgAppResp Term:1 Log:0/15 Commit:15 +3->1 MsgAppResp Term:1 Log:0/15 Commit:15 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] diff --git a/testdata/async_storage_writes_append_aba_race.txt b/testdata/async_storage_writes_append_aba_race.txt index 4b68330f..399800e8 100644 --- a/testdata/async_storage_writes_append_aba_race.txt +++ b/testdata/async_storage_writes_append_aba_race.txt @@ -42,7 +42,7 @@ Messages: 2->5 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] 2->6 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] 2->7 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] -2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[2->2 MsgAppResp Term:1 Log:0/12, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12] +2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[2->2 MsgAppResp Term:1 Log:0/12 Commit:11, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12] deliver-msgs 1 drop=(3,4,5,6,7) ---- @@ -61,7 +61,7 @@ Ready MustSync=true: Entries: 1/12 EntryNormal "init_prop" Messages: -1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[1->2 MsgAppResp Term:1 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12] +1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[1->2 MsgAppResp Term:1 Log:0/12 Commit:11, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12] # Step 4: node 3 becomes the leader after getting a vote from nodes 4, 5, and 6. @@ -183,7 +183,7 @@ Messages: 3->5 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] 3->6 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] 3->7 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] -3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[2/12 EntryNormal ""] Responses:[3->3 MsgAppResp Term:2 Log:0/12, AppendThread->3 MsgStorageAppendResp Term:2 Log:2/12] +3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[2/12 EntryNormal ""] Responses:[3->3 MsgAppResp Term:2 Log:0/12 Commit:11, AppendThread->3 MsgStorageAppendResp Term:2 Log:2/12] deliver-msgs 1 drop=(2,4,5,6,7) ---- @@ -210,7 +210,7 @@ HardState Term:2 Commit:11 Entries: 2/12 EntryNormal "" Messages: -1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] Responses:[1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0), 1->3 MsgAppResp Term:2 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12] +1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] Responses:[1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0), 1->3 MsgAppResp Term:2 Log:0/12 Commit:11, AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12] # Step 6: node 3 crashes and node 4 becomes leader getting the vote from 5, 6, and 7. @@ -325,7 +325,7 @@ Messages: 4->5 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] 4->6 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] 4->7 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] -4->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[4->4 MsgAppResp Term:3 Log:0/12, AppendThread->4 MsgStorageAppendResp Term:3 Log:3/12] +4->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[4->4 MsgAppResp Term:3 Log:0/12 Commit:11, AppendThread->4 MsgStorageAppendResp Term:3 Log:3/12] # Step 7: before the new entries reach node 1, it hears of the term change # through a heartbeat and persists the new term. Node 1 then receives these @@ -390,7 +390,7 @@ Ready MustSync=true: Entries: 3/12 EntryNormal "" Messages: -1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[1->4 MsgAppResp Term:3 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12] +1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[1->4 MsgAppResp Term:3 Log:0/12 Commit:11, AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12] # Step 8: The asynchronous log appends from the first Ready complete and the # MsgStorageAppendResp is returned to the raft node state machine. A decision @@ -405,7 +405,7 @@ process-append-thread 1 Processing: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses: -1->2 MsgAppResp Term:1 Log:0/12 +1->2 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 raft-log 1 @@ -432,7 +432,7 @@ Processing: 1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] Responses: 1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0) -1->3 MsgAppResp Term:2 Log:0/12 +1->3 MsgAppResp Term:2 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12 raft-log 1 @@ -467,7 +467,7 @@ process-append-thread 1 Processing: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses: -1->4 MsgAppResp Term:3 Log:0/12 +1->4 MsgAppResp Term:3 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12 raft-log 1 diff --git a/testdata/campaign.txt b/testdata/campaign.txt index 4eefab36..3538aa64 100644 --- a/testdata/campaign.txt +++ b/testdata/campaign.txt @@ -76,17 +76,17 @@ stabilize Entries: 1/3 EntryNormal "" Messages: - 2->1 MsgAppResp Term:1 Log:0/3 + 2->1 MsgAppResp Term:1 Log:0/3 Commit:2 > 3 handling Ready Ready MustSync=true: Lead:1 State:StateFollower Entries: 1/3 EntryNormal "" Messages: - 3->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgAppResp Term:1 Log:0/3 Commit:2 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/3 - 3->1 MsgAppResp Term:1 Log:0/3 + 2->1 MsgAppResp Term:1 Log:0/3 Commit:2 + 3->1 MsgAppResp Term:1 Log:0/3 Commit:2 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:3 @@ -105,14 +105,14 @@ stabilize CommittedEntries: 1/3 EntryNormal "" Messages: - 2->1 MsgAppResp Term:1 Log:0/3 + 2->1 MsgAppResp Term:1 Log:0/3 Commit:3 > 3 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:3 CommittedEntries: 1/3 EntryNormal "" Messages: - 3->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgAppResp Term:1 Log:0/3 Commit:3 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/3 - 3->1 MsgAppResp Term:1 Log:0/3 + 2->1 MsgAppResp Term:1 Log:0/3 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/3 Commit:3 diff --git a/testdata/campaign_learner_must_vote.txt b/testdata/campaign_learner_must_vote.txt index 85617df3..734422b8 100644 --- a/testdata/campaign_learner_must_vote.txt +++ b/testdata/campaign_learner_must_vote.txt @@ -129,10 +129,10 @@ stabilize 2 3 CommittedEntries: 1/4 EntryConfChangeV2 v3 Messages: - 3->2 MsgAppResp Term:2 Log:0/5 + 3->2 MsgAppResp Term:2 Log:0/5 Commit:4 INFO 3 switched to configuration voters=(1 2 3) > 2 receiving messages - 3->2 MsgAppResp Term:2 Log:0/5 + 3->2 MsgAppResp Term:2 Log:0/5 Commit:4 > 2 handling Ready Ready MustSync=false: HardState Term:2 Vote:2 Commit:5 @@ -148,6 +148,6 @@ stabilize 2 3 CommittedEntries: 2/5 EntryNormal "" Messages: - 3->2 MsgAppResp Term:2 Log:0/5 + 3->2 MsgAppResp Term:2 Log:0/5 Commit:5 > 2 receiving messages - 3->2 MsgAppResp Term:2 Log:0/5 + 3->2 MsgAppResp Term:2 Log:0/5 Commit:5 diff --git a/testdata/checkquorum.txt b/testdata/checkquorum.txt index b25c1e63..065e0754 100644 --- a/testdata/checkquorum.txt +++ b/testdata/checkquorum.txt @@ -193,7 +193,7 @@ stabilize Entries: 3/12 EntryNormal "" Messages: - 1->2 MsgAppResp Term:3 Log:0/12 + 1->2 MsgAppResp Term:3 Log:0/12 Commit:11 > 3 handling Ready Ready MustSync=true: Lead:2 State:StateFollower @@ -201,10 +201,10 @@ stabilize Entries: 3/12 EntryNormal "" Messages: - 3->2 MsgAppResp Term:3 Log:0/12 + 3->2 MsgAppResp Term:3 Log:0/12 Commit:11 > 2 receiving messages - 1->2 MsgAppResp Term:3 Log:0/12 - 3->2 MsgAppResp Term:3 Log:0/12 + 1->2 MsgAppResp Term:3 Log:0/12 Commit:11 + 3->2 MsgAppResp Term:3 Log:0/12 Commit:11 > 2 handling Ready Ready MustSync=false: HardState Term:3 Vote:2 Commit:12 @@ -223,14 +223,14 @@ stabilize CommittedEntries: 3/12 EntryNormal "" Messages: - 1->2 MsgAppResp Term:3 Log:0/12 + 1->2 MsgAppResp Term:3 Log:0/12 Commit:12 > 3 handling Ready Ready MustSync=false: HardState Term:3 Commit:12 CommittedEntries: 3/12 EntryNormal "" Messages: - 3->2 MsgAppResp Term:3 Log:0/12 + 3->2 MsgAppResp Term:3 Log:0/12 Commit:12 > 2 receiving messages - 1->2 MsgAppResp Term:3 Log:0/12 - 3->2 MsgAppResp Term:3 Log:0/12 + 1->2 MsgAppResp Term:3 Log:0/12 Commit:12 + 3->2 MsgAppResp Term:3 Log:0/12 Commit:12 diff --git a/testdata/confchange_v1_add_single.txt b/testdata/confchange_v1_add_single.txt index 0419d28d..e9469ed7 100644 --- a/testdata/confchange_v1_add_single.txt +++ b/testdata/confchange_v1_add_single.txt @@ -101,6 +101,6 @@ stabilize > 2 handling Ready Ready MustSync=false: Messages: - 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 diff --git a/testdata/confchange_v1_remove_leader.txt b/testdata/confchange_v1_remove_leader.txt index cc91508a..fde99c82 100644 --- a/testdata/confchange_v1_remove_leader.txt +++ b/testdata/confchange_v1_remove_leader.txt @@ -70,8 +70,8 @@ stabilize 2 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 2->1 MsgAppResp Term:1 Log:0/4 - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:3 # Put another entry in n1's log. propose 1 bar @@ -92,8 +92,8 @@ stabilize 1 1->2 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] 1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:3 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:5 @@ -129,18 +129,18 @@ stabilize 2 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 INFO 2 switched to configuration voters=(2 3) # ... which thankfully is what we see on the leader. stabilize 1 ---- > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 # When n3 responds, quorum is reached and everything falls into place. stabilize @@ -162,18 +162,18 @@ stabilize 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 3->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 INFO 3 switched to configuration voters=(2 3) > 1 receiving messages - 3->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:6 @@ -192,17 +192,17 @@ stabilize CommittedEntries: 1/6 EntryNormal "bar" Messages: - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:6 > 3 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:6 CommittedEntries: 1/6 EntryNormal "bar" Messages: - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:6 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:6 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:6 # However not all is well. n1 is still leader but unconditionally drops all # proposals on the floor, so we're effectively stuck if it still heartbeats diff --git a/testdata/confchange_v1_remove_leader_stepdown.txt b/testdata/confchange_v1_remove_leader_stepdown.txt index fe397650..b9e14ff0 100644 --- a/testdata/confchange_v1_remove_leader_stepdown.txt +++ b/testdata/confchange_v1_remove_leader_stepdown.txt @@ -71,8 +71,8 @@ stabilize 2 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 2->1 MsgAppResp Term:1 Log:0/4 - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:3 # Put another entry in n1's log. propose 1 bar @@ -91,8 +91,8 @@ stabilize 1 1->2 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] 1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:3 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:5 @@ -132,18 +132,18 @@ stabilize 2 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 INFO 2 switched to configuration voters=(2 3) # ...because the old leader n1 ignores the append responses. stabilize 1 ---- > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 # When n3 responds, quorum is reached and everything falls into place. stabilize @@ -165,18 +165,18 @@ stabilize 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 3->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 INFO 3 switched to configuration voters=(2 3) > 1 receiving messages - 3->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 # n1 can no longer propose. propose 1 baz diff --git a/testdata/confchange_v2_add_double_auto.txt b/testdata/confchange_v2_add_double_auto.txt index 5cf4f24c..78b5a2bc 100644 --- a/testdata/confchange_v2_add_double_auto.txt +++ b/testdata/confchange_v2_add_double_auto.txt @@ -126,9 +126,9 @@ stabilize 1 2 Entries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:5 @@ -145,10 +145,10 @@ stabilize 1 2 CommittedEntries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 INFO 2 switched to configuration voters=(1 2 3) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 # n3 immediately receives a snapshot in the final configuration. stabilize 1 3 @@ -198,9 +198,9 @@ stabilize 1 3 > 3 handling Ready Ready MustSync=false: Messages: - 3->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:5 > 1 receiving messages - 3->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:5 # Nothing else happens. stabilize @@ -238,13 +238,13 @@ stabilize 2 3 Entries: 1/6 EntryConfChangeV2 r2 r3 Messages: - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 > 3 handling Ready Ready MustSync=true: Entries: 1/6 EntryConfChangeV2 r2 r3 Messages: - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 # n1 gets some more proposals. This is part of a regression test: There used to # be a bug in which these proposals would prompt the leader to transition out of @@ -272,8 +272,8 @@ stabilize 1 1->2 MsgApp Term:1 Log:1/7 Commit:5 Entries:[1/8 EntryNormal "bar"] 1->3 MsgApp Term:1 Log:1/7 Commit:5 Entries:[1/8 EntryNormal "bar"] > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:6 @@ -315,10 +315,10 @@ stabilize 2 3 CommittedEntries: 1/6 EntryConfChangeV2 r2 r3 Messages: - 2->1 MsgAppResp Term:1 Log:0/7 - 2->1 MsgAppResp Term:1 Log:0/8 - 2->1 MsgAppResp Term:1 Log:0/8 - 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/8 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/8 Commit:6 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:6 INFO 2 switched to configuration voters=(1)&&(1 2 3) autoleave > 3 handling Ready Ready MustSync=true: @@ -330,10 +330,10 @@ stabilize 2 3 CommittedEntries: 1/6 EntryConfChangeV2 r2 r3 Messages: - 3->1 MsgAppResp Term:1 Log:0/7 - 3->1 MsgAppResp Term:1 Log:0/8 - 3->1 MsgAppResp Term:1 Log:0/8 - 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/7 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/8 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/8 Commit:6 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:6 INFO 3 switched to configuration voters=(1)&&(1 2 3) autoleave # n2 and n3 also leave the joint config and the dust settles. We see at the very @@ -342,14 +342,14 @@ stabilize 2 3 stabilize ---- > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/7 - 2->1 MsgAppResp Term:1 Log:0/8 - 2->1 MsgAppResp Term:1 Log:0/8 - 2->1 MsgAppResp Term:1 Log:0/9 - 3->1 MsgAppResp Term:1 Log:0/7 - 3->1 MsgAppResp Term:1 Log:0/8 - 3->1 MsgAppResp Term:1 Log:0/8 - 3->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/8 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/8 Commit:6 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:6 + 3->1 MsgAppResp Term:1 Log:0/7 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/8 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/8 Commit:6 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:6 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:9 @@ -381,9 +381,9 @@ stabilize 1/8 EntryNormal "bar" 1/9 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/9 - 2->1 MsgAppResp Term:1 Log:0/9 - 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:7 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:8 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:9 INFO 2 switched to configuration voters=(1) > 3 handling Ready Ready MustSync=false: @@ -393,20 +393,20 @@ stabilize 1/8 EntryNormal "bar" 1/9 EntryConfChangeV2 Messages: - 3->1 MsgAppResp Term:1 Log:0/9 - 3->1 MsgAppResp Term:1 Log:0/9 - 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:7 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:8 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:9 INFO 3 switched to configuration voters=(1) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:7 raft: cannot step as peer not found - 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:8 raft: cannot step as peer not found - 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:9 raft: cannot step as peer not found - 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:7 raft: cannot step as peer not found - 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:8 raft: cannot step as peer not found - 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:9 raft: cannot step as peer not found diff --git a/testdata/confchange_v2_add_double_implicit.txt b/testdata/confchange_v2_add_double_implicit.txt index 81b3f0de..7d27151f 100644 --- a/testdata/confchange_v2_add_double_implicit.txt +++ b/testdata/confchange_v2_add_double_implicit.txt @@ -109,9 +109,9 @@ stabilize 1 2 Entries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:5 @@ -128,7 +128,7 @@ stabilize 1 2 CommittedEntries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 INFO 2 switched to configuration voters=(1 2) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 diff --git a/testdata/confchange_v2_add_single_auto.txt b/testdata/confchange_v2_add_single_auto.txt index fe127bf5..666ddb52 100644 --- a/testdata/confchange_v2_add_single_auto.txt +++ b/testdata/confchange_v2_add_single_auto.txt @@ -102,6 +102,6 @@ stabilize > 2 handling Ready Ready MustSync=false: Messages: - 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 diff --git a/testdata/confchange_v2_add_single_explicit.txt b/testdata/confchange_v2_add_single_explicit.txt index c51900f0..4c83d5f1 100644 --- a/testdata/confchange_v2_add_single_explicit.txt +++ b/testdata/confchange_v2_add_single_explicit.txt @@ -102,9 +102,9 @@ stabilize 1 2 > 2 handling Ready Ready MustSync=false: Messages: - 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 # Check that we're not allowed to change membership again while in the joint state. # This leads to an empty entry being proposed instead (index 5 in the stabilize block @@ -139,11 +139,11 @@ stabilize 1/5 EntryNormal "" 1/6 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:6 @@ -164,12 +164,12 @@ stabilize 1/5 EntryNormal "" 1/6 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:6 INFO 2 switched to configuration voters=(1 2) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:6 # Check that trying to transition out again won't do anything. propose-conf-change 1 @@ -192,9 +192,9 @@ stabilize Entries: 1/7 EntryNormal "" Messages: - 2->1 MsgAppResp Term:1 Log:0/7 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:6 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/7 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:6 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:7 @@ -210,6 +210,6 @@ stabilize CommittedEntries: 1/7 EntryNormal "" Messages: - 2->1 MsgAppResp Term:1 Log:0/7 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:7 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/7 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:7 diff --git a/testdata/confchange_v2_replace_leader.txt b/testdata/confchange_v2_replace_leader.txt index dfb09505..4e2bf5cb 100644 --- a/testdata/confchange_v2_replace_leader.txt +++ b/testdata/confchange_v2_replace_leader.txt @@ -65,16 +65,16 @@ stabilize Entries: 1/4 EntryConfChangeV2 r1 v4 Messages: - 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 > 3 handling Ready Ready MustSync=true: Entries: 1/4 EntryConfChangeV2 r1 v4 Messages: - 3->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:4 @@ -98,7 +98,7 @@ stabilize CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: - 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 INFO 2 switched to configuration voters=(2 3 4)&&(1 2 3) > 3 handling Ready Ready MustSync=false: @@ -106,11 +106,11 @@ stabilize CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: - 3->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:4 INFO 3 switched to configuration voters=(2 3 4)&&(1 2 3) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 4 receiving messages 1->4 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 r1 v4] INFO 4 [term: 0] received a MsgApp message with higher term from 1 [term: 1] @@ -150,9 +150,9 @@ stabilize > 4 handling Ready Ready MustSync=false: Messages: - 4->1 MsgAppResp Term:1 Log:0/4 + 4->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 1 receiving messages - 4->1 MsgAppResp Term:1 Log:0/4 + 4->1 MsgAppResp Term:1 Log:0/4 Commit:4 # Transfer leadership while in the joint config. @@ -257,25 +257,25 @@ stabilize Entries: 2/5 EntryNormal "" Messages: - 1->4 MsgAppResp Term:2 Log:0/5 + 1->4 MsgAppResp Term:2 Log:0/5 Commit:4 > 2 handling Ready Ready MustSync=true: Lead:4 State:StateFollower Entries: 2/5 EntryNormal "" Messages: - 2->4 MsgAppResp Term:2 Log:0/5 + 2->4 MsgAppResp Term:2 Log:0/5 Commit:4 > 3 handling Ready Ready MustSync=true: Lead:4 State:StateFollower Entries: 2/5 EntryNormal "" Messages: - 3->4 MsgAppResp Term:2 Log:0/5 + 3->4 MsgAppResp Term:2 Log:0/5 Commit:4 > 4 receiving messages - 1->4 MsgAppResp Term:2 Log:0/5 - 2->4 MsgAppResp Term:2 Log:0/5 - 3->4 MsgAppResp Term:2 Log:0/5 + 1->4 MsgAppResp Term:2 Log:0/5 Commit:4 + 2->4 MsgAppResp Term:2 Log:0/5 Commit:4 + 3->4 MsgAppResp Term:2 Log:0/5 Commit:4 > 4 handling Ready Ready MustSync=false: HardState Term:2 Vote:4 Commit:5 @@ -299,27 +299,27 @@ stabilize CommittedEntries: 2/5 EntryNormal "" Messages: - 1->4 MsgAppResp Term:2 Log:0/5 - 1->4 MsgAppResp Term:2 Log:0/5 + 1->4 MsgAppResp Term:2 Log:0/5 Commit:4 + 1->4 MsgAppResp Term:2 Log:0/5 Commit:5 > 2 handling Ready Ready MustSync=false: HardState Term:2 Vote:4 Commit:5 CommittedEntries: 2/5 EntryNormal "" Messages: - 2->4 MsgAppResp Term:2 Log:0/5 + 2->4 MsgAppResp Term:2 Log:0/5 Commit:5 > 3 handling Ready Ready MustSync=false: HardState Term:2 Vote:4 Commit:5 CommittedEntries: 2/5 EntryNormal "" Messages: - 3->4 MsgAppResp Term:2 Log:0/5 + 3->4 MsgAppResp Term:2 Log:0/5 Commit:5 > 4 receiving messages - 1->4 MsgAppResp Term:2 Log:0/5 - 1->4 MsgAppResp Term:2 Log:0/5 - 2->4 MsgAppResp Term:2 Log:0/5 - 3->4 MsgAppResp Term:2 Log:0/5 + 1->4 MsgAppResp Term:2 Log:0/5 Commit:4 + 1->4 MsgAppResp Term:2 Log:0/5 Commit:5 + 2->4 MsgAppResp Term:2 Log:0/5 Commit:5 + 3->4 MsgAppResp Term:2 Log:0/5 Commit:5 # Leadership transfer succeeded. raft-state @@ -356,23 +356,23 @@ stabilize Entries: 2/6 EntryConfChangeV2 Messages: - 1->4 MsgAppResp Term:2 Log:0/6 + 1->4 MsgAppResp Term:2 Log:0/6 Commit:5 > 2 handling Ready Ready MustSync=true: Entries: 2/6 EntryConfChangeV2 Messages: - 2->4 MsgAppResp Term:2 Log:0/6 + 2->4 MsgAppResp Term:2 Log:0/6 Commit:5 > 3 handling Ready Ready MustSync=true: Entries: 2/6 EntryConfChangeV2 Messages: - 3->4 MsgAppResp Term:2 Log:0/6 + 3->4 MsgAppResp Term:2 Log:0/6 Commit:5 > 4 receiving messages - 1->4 MsgAppResp Term:2 Log:0/6 - 2->4 MsgAppResp Term:2 Log:0/6 - 3->4 MsgAppResp Term:2 Log:0/6 + 1->4 MsgAppResp Term:2 Log:0/6 Commit:5 + 2->4 MsgAppResp Term:2 Log:0/6 Commit:5 + 3->4 MsgAppResp Term:2 Log:0/6 Commit:5 > 4 handling Ready Ready MustSync=false: HardState Term:2 Vote:4 Commit:6 @@ -395,7 +395,7 @@ stabilize CommittedEntries: 2/6 EntryConfChangeV2 Messages: - 1->4 MsgAppResp Term:2 Log:0/6 + 1->4 MsgAppResp Term:2 Log:0/6 Commit:6 INFO 1 switched to configuration voters=(2 3 4) > 2 handling Ready Ready MustSync=false: @@ -403,7 +403,7 @@ stabilize CommittedEntries: 2/6 EntryConfChangeV2 Messages: - 2->4 MsgAppResp Term:2 Log:0/6 + 2->4 MsgAppResp Term:2 Log:0/6 Commit:6 INFO 2 switched to configuration voters=(2 3 4) > 3 handling Ready Ready MustSync=false: @@ -411,13 +411,13 @@ stabilize CommittedEntries: 2/6 EntryConfChangeV2 Messages: - 3->4 MsgAppResp Term:2 Log:0/6 + 3->4 MsgAppResp Term:2 Log:0/6 Commit:6 INFO 3 switched to configuration voters=(2 3 4) > 4 receiving messages - 1->4 MsgAppResp Term:2 Log:0/6 + 1->4 MsgAppResp Term:2 Log:0/6 Commit:6 raft: cannot step as peer not found - 2->4 MsgAppResp Term:2 Log:0/6 - 3->4 MsgAppResp Term:2 Log:0/6 + 2->4 MsgAppResp Term:2 Log:0/6 Commit:6 + 3->4 MsgAppResp Term:2 Log:0/6 Commit:6 # n1 is out of the configuration. raft-state diff --git a/testdata/confchange_v2_replace_leader_stepdown.txt b/testdata/confchange_v2_replace_leader_stepdown.txt index 62d01d23..ecad1d7c 100644 --- a/testdata/confchange_v2_replace_leader_stepdown.txt +++ b/testdata/confchange_v2_replace_leader_stepdown.txt @@ -93,23 +93,23 @@ stabilize Entries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 3 handling Ready Ready MustSync=true: Entries: 1/5 EntryConfChangeV2 Messages: - 3->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 4 handling Ready Ready MustSync=true: Entries: 1/5 EntryConfChangeV2 Messages: - 4->1 MsgAppResp Term:1 Log:0/5 + 4->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/5 - 4->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:4 + 4->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:5 @@ -136,7 +136,7 @@ stabilize CommittedEntries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 INFO 2 switched to configuration voters=(2 3 4) > 3 handling Ready Ready MustSync=false: @@ -144,7 +144,7 @@ stabilize CommittedEntries: 1/5 EntryConfChangeV2 Messages: - 3->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:5 INFO 3 switched to configuration voters=(2 3 4) > 4 handling Ready Ready MustSync=false: @@ -152,12 +152,12 @@ stabilize CommittedEntries: 1/5 EntryConfChangeV2 Messages: - 4->1 MsgAppResp Term:1 Log:0/5 + 4->1 MsgAppResp Term:1 Log:0/5 Commit:5 INFO 4 switched to configuration voters=(2 3 4) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/5 - 4->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:5 + 4->1 MsgAppResp Term:1 Log:0/5 Commit:5 # n1 is out of the configuration. raft-state diff --git a/testdata/forget_leader_prevote_checkquorum.txt b/testdata/forget_leader_prevote_checkquorum.txt index 9b3b80ff..d204b9d7 100644 --- a/testdata/forget_leader_prevote_checkquorum.txt +++ b/testdata/forget_leader_prevote_checkquorum.txt @@ -182,7 +182,7 @@ stabilize 2 Entries: 2/13 EntryNormal "prop_1" Messages: - 2->3 MsgAppResp Term:2 Log:0/13 + 2->3 MsgAppResp Term:2 Log:0/13 Commit:12 forget-leader 2 ---- diff --git a/testdata/heartbeat_resp_recovers_from_probing.txt b/testdata/heartbeat_resp_recovers_from_probing.txt index e606a155..bb6eef1e 100644 --- a/testdata/heartbeat_resp_recovers_from_probing.txt +++ b/testdata/heartbeat_resp_recovers_from_probing.txt @@ -79,9 +79,9 @@ stabilize > 2 handling Ready Ready MustSync=false: Messages: - 2->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:11 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:11 status 1 ---- diff --git a/testdata/prevote.txt b/testdata/prevote.txt index db763d35..f4926d43 100644 --- a/testdata/prevote.txt +++ b/testdata/prevote.txt @@ -49,7 +49,7 @@ Ready MustSync=true: Entries: 1/12 EntryNormal "prop_1" Messages: -2->1 MsgAppResp Term:1 Log:0/12 +2->1 MsgAppResp Term:1 Log:0/12 Commit:11 # 3 is now behind on its log. Attempt to campaign. raft-log 3 @@ -75,7 +75,7 @@ INFO 3 has received 1 MsgPreVoteResp votes and 0 vote rejections deliver-msgs 1 2 ---- -2->1 MsgAppResp Term:1 Log:0/12 +2->1 MsgAppResp Term:1 Log:0/12 Commit:11 3->1 MsgPreVote Term:2 Log:1/11 INFO 1 [logterm: 1, index: 12, vote: 1] rejected MsgPreVote from 3 [logterm: 1, index: 11] at term 1 3->2 MsgPreVote Term:2 Log:1/11 @@ -111,7 +111,7 @@ stabilize CommittedEntries: 1/12 EntryNormal "prop_1" Messages: - 2->1 MsgAppResp Term:1 Log:0/12 + 2->1 MsgAppResp Term:1 Log:0/12 Commit:12 > 3 handling Ready Ready MustSync=true: Lead:1 State:StateFollower @@ -121,12 +121,12 @@ stabilize CommittedEntries: 1/12 EntryNormal "prop_1" Messages: - 3->1 MsgAppResp Term:1 Log:0/12 - 3->1 MsgAppResp Term:1 Log:0/12 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:11 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:12 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/12 - 3->1 MsgAppResp Term:1 Log:0/12 - 3->1 MsgAppResp Term:1 Log:0/12 + 2->1 MsgAppResp Term:1 Log:0/12 Commit:12 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:11 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:12 # Let 2 campaign. It should succeed, since it's up-to-date on the log. campaign 2 @@ -223,17 +223,17 @@ stabilize Entries: 2/13 EntryNormal "" Messages: - 1->2 MsgAppResp Term:2 Log:0/13 + 1->2 MsgAppResp Term:2 Log:0/13 Commit:12 > 3 handling Ready Ready MustSync=true: Lead:2 State:StateFollower Entries: 2/13 EntryNormal "" Messages: - 3->2 MsgAppResp Term:2 Log:0/13 + 3->2 MsgAppResp Term:2 Log:0/13 Commit:12 > 2 receiving messages - 1->2 MsgAppResp Term:2 Log:0/13 - 3->2 MsgAppResp Term:2 Log:0/13 + 1->2 MsgAppResp Term:2 Log:0/13 Commit:12 + 3->2 MsgAppResp Term:2 Log:0/13 Commit:12 > 2 handling Ready Ready MustSync=false: HardState Term:2 Vote:2 Commit:13 @@ -252,14 +252,14 @@ stabilize CommittedEntries: 2/13 EntryNormal "" Messages: - 1->2 MsgAppResp Term:2 Log:0/13 + 1->2 MsgAppResp Term:2 Log:0/13 Commit:13 > 3 handling Ready Ready MustSync=false: HardState Term:2 Vote:2 Commit:13 CommittedEntries: 2/13 EntryNormal "" Messages: - 3->2 MsgAppResp Term:2 Log:0/13 + 3->2 MsgAppResp Term:2 Log:0/13 Commit:13 > 2 receiving messages - 1->2 MsgAppResp Term:2 Log:0/13 - 3->2 MsgAppResp Term:2 Log:0/13 + 1->2 MsgAppResp Term:2 Log:0/13 Commit:13 + 3->2 MsgAppResp Term:2 Log:0/13 Commit:13 diff --git a/testdata/prevote_checkquorum.txt b/testdata/prevote_checkquorum.txt index 6db6662b..a7cc57ae 100644 --- a/testdata/prevote_checkquorum.txt +++ b/testdata/prevote_checkquorum.txt @@ -147,17 +147,17 @@ stabilize Entries: 2/12 EntryNormal "" Messages: - 1->3 MsgAppResp Term:2 Log:0/12 + 1->3 MsgAppResp Term:2 Log:0/12 Commit:11 > 2 handling Ready Ready MustSync=true: Lead:3 State:StateFollower Entries: 2/12 EntryNormal "" Messages: - 2->3 MsgAppResp Term:2 Log:0/12 + 2->3 MsgAppResp Term:2 Log:0/12 Commit:11 > 3 receiving messages - 1->3 MsgAppResp Term:2 Log:0/12 - 2->3 MsgAppResp Term:2 Log:0/12 + 1->3 MsgAppResp Term:2 Log:0/12 Commit:11 + 2->3 MsgAppResp Term:2 Log:0/12 Commit:11 > 3 handling Ready Ready MustSync=false: HardState Term:2 Vote:3 Commit:12 @@ -176,17 +176,17 @@ stabilize CommittedEntries: 2/12 EntryNormal "" Messages: - 1->3 MsgAppResp Term:2 Log:0/12 + 1->3 MsgAppResp Term:2 Log:0/12 Commit:12 > 2 handling Ready Ready MustSync=false: HardState Term:2 Vote:3 Commit:12 CommittedEntries: 2/12 EntryNormal "" Messages: - 2->3 MsgAppResp Term:2 Log:0/12 + 2->3 MsgAppResp Term:2 Log:0/12 Commit:12 > 3 receiving messages - 1->3 MsgAppResp Term:2 Log:0/12 - 2->3 MsgAppResp Term:2 Log:0/12 + 1->3 MsgAppResp Term:2 Log:0/12 Commit:12 + 2->3 MsgAppResp Term:2 Log:0/12 Commit:12 # Node 3 is now the leader. Even though the leader is active, nodes 1 and 2 can # still win a prevote and election if they both explicitly campaign, since the @@ -302,7 +302,7 @@ stabilize Entries: 3/13 EntryNormal "" Messages: - 1->2 MsgAppResp Term:3 Log:0/13 + 1->2 MsgAppResp Term:3 Log:0/13 Commit:12 > 3 handling Ready Ready MustSync=true: Lead:2 State:StateFollower @@ -310,10 +310,10 @@ stabilize Entries: 3/13 EntryNormal "" Messages: - 3->2 MsgAppResp Term:3 Log:0/13 + 3->2 MsgAppResp Term:3 Log:0/13 Commit:12 > 2 receiving messages - 1->2 MsgAppResp Term:3 Log:0/13 - 3->2 MsgAppResp Term:3 Log:0/13 + 1->2 MsgAppResp Term:3 Log:0/13 Commit:12 + 3->2 MsgAppResp Term:3 Log:0/13 Commit:12 > 2 handling Ready Ready MustSync=false: HardState Term:3 Vote:2 Commit:13 @@ -332,14 +332,14 @@ stabilize CommittedEntries: 3/13 EntryNormal "" Messages: - 1->2 MsgAppResp Term:3 Log:0/13 + 1->2 MsgAppResp Term:3 Log:0/13 Commit:13 > 3 handling Ready Ready MustSync=false: HardState Term:3 Commit:13 CommittedEntries: 3/13 EntryNormal "" Messages: - 3->2 MsgAppResp Term:3 Log:0/13 + 3->2 MsgAppResp Term:3 Log:0/13 Commit:13 > 2 receiving messages - 1->2 MsgAppResp Term:3 Log:0/13 - 3->2 MsgAppResp Term:3 Log:0/13 + 1->2 MsgAppResp Term:3 Log:0/13 Commit:13 + 3->2 MsgAppResp Term:3 Log:0/13 Commit:13 diff --git a/testdata/probe_and_replicate.txt b/testdata/probe_and_replicate.txt index d5829704..715417ed 100644 --- a/testdata/probe_and_replicate.txt +++ b/testdata/probe_and_replicate.txt @@ -504,9 +504,9 @@ stabilize 1 2 6/20 EntryNormal "prop_6_20" 8/21 EntryNormal "" Messages: - 2->1 MsgAppResp Term:8 Log:0/21 + 2->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 receiving messages - 2->1 MsgAppResp Term:8 Log:0/21 + 2->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 handling Ready Ready MustSync=false: Messages: @@ -516,9 +516,9 @@ stabilize 1 2 > 2 handling Ready Ready MustSync=false: Messages: - 2->1 MsgAppResp Term:8 Log:0/21 + 2->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 receiving messages - 2->1 MsgAppResp Term:8 Log:0/21 + 2->1 MsgAppResp Term:8 Log:0/21 Commit:18 stabilize 1 3 ---- @@ -554,9 +554,9 @@ stabilize 1 3 5/17 EntryNormal "prop_5_17" 6/18 EntryNormal "" Messages: - 3->1 MsgAppResp Term:8 Log:0/21 + 3->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 receiving messages - 3->1 MsgAppResp Term:8 Log:0/21 + 3->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 handling Ready Ready MustSync=false: Messages: @@ -566,9 +566,9 @@ stabilize 1 3 > 3 handling Ready Ready MustSync=false: Messages: - 3->1 MsgAppResp Term:8 Log:0/21 + 3->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 receiving messages - 3->1 MsgAppResp Term:8 Log:0/21 + 3->1 MsgAppResp Term:8 Log:0/21 Commit:18 stabilize 1 4 ---- @@ -582,9 +582,9 @@ stabilize 1 4 Entries: 8/21 EntryNormal "" Messages: - 4->1 MsgAppResp Term:8 Log:0/21 + 4->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 receiving messages - 4->1 MsgAppResp Term:8 Log:0/21 + 4->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 handling Ready Ready MustSync=false: HardState Term:8 Vote:1 Commit:21 @@ -606,9 +606,9 @@ stabilize 1 4 6/20 EntryNormal "prop_6_20" 8/21 EntryNormal "" Messages: - 4->1 MsgAppResp Term:8 Log:0/21 + 4->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 4->1 MsgAppResp Term:8 Log:0/21 + 4->1 MsgAppResp Term:8 Log:0/21 Commit:21 stabilize 1 5 ---- @@ -641,9 +641,9 @@ stabilize 1 5 6/20 EntryNormal "prop_6_20" 8/21 EntryNormal "" Messages: - 5->1 MsgAppResp Term:8 Log:0/21 + 5->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 5->1 MsgAppResp Term:8 Log:0/21 + 5->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 handling Ready Ready MustSync=false: Messages: @@ -653,9 +653,9 @@ stabilize 1 5 > 5 handling Ready Ready MustSync=false: Messages: - 5->1 MsgAppResp Term:8 Log:0/21 + 5->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 5->1 MsgAppResp Term:8 Log:0/21 + 5->1 MsgAppResp Term:8 Log:0/21 Commit:21 stabilize 1 6 ---- @@ -694,9 +694,9 @@ stabilize 1 6 6/20 EntryNormal "prop_6_20" 8/21 EntryNormal "" Messages: - 6->1 MsgAppResp Term:8 Log:0/21 + 6->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 6->1 MsgAppResp Term:8 Log:0/21 + 6->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 handling Ready Ready MustSync=false: Messages: @@ -706,9 +706,9 @@ stabilize 1 6 > 6 handling Ready Ready MustSync=false: Messages: - 6->1 MsgAppResp Term:8 Log:0/21 + 6->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 6->1 MsgAppResp Term:8 Log:0/21 + 6->1 MsgAppResp Term:8 Log:0/21 Commit:21 stabilize 1 7 ---- @@ -751,9 +751,9 @@ stabilize 1 7 6/20 EntryNormal "prop_6_20" 8/21 EntryNormal "" Messages: - 7->1 MsgAppResp Term:8 Log:0/21 + 7->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 7->1 MsgAppResp Term:8 Log:0/21 + 7->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 handling Ready Ready MustSync=false: Messages: @@ -763,6 +763,6 @@ stabilize 1 7 > 7 handling Ready Ready MustSync=false: Messages: - 7->1 MsgAppResp Term:8 Log:0/21 + 7->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 7->1 MsgAppResp Term:8 Log:0/21 + 7->1 MsgAppResp Term:8 Log:0/21 Commit:21 diff --git a/testdata/snapshot_succeed_via_app_resp.txt b/testdata/snapshot_succeed_via_app_resp.txt index 80ed3646..37917403 100644 --- a/testdata/snapshot_succeed_via_app_resp.txt +++ b/testdata/snapshot_succeed_via_app_resp.txt @@ -150,7 +150,7 @@ stabilize > 3 handling Ready Ready MustSync=false: Messages: - 3->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:11 > 1 receiving messages 2->1 MsgHeartbeatResp Term:1 Log:0/0 - 3->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:11 diff --git a/tracker/progress.go b/tracker/progress.go index b4d07deb..e80059f5 100644 --- a/tracker/progress.go +++ b/tracker/progress.go @@ -30,9 +30,13 @@ import ( type Progress struct { // Match is log index up to which the follower's log matches the leader's. Match uint64 + // Commit is the commit index of the follower's log. + // INVARIANT: Commit <= Match. + Commit uint64 // Next is the index of the next log entry to be sent to the follower. Entries // in the (Match, Next) interval are on the fly to this follower. Next uint64 + // State defines how the leader should interact with the follower. // // When in StateProbe, leader sends at most one replication message @@ -180,6 +184,13 @@ func (pr *Progress) UpdateOnEntriesSend(entries int, bytes, nextIndex uint64) er return nil } +// UpdateCommit moves the known commit index for this follower forward. +func (pr *Progress) UpdateCommit(index uint64) { + if index > pr.Commit { + pr.Commit = index + } +} + // MaybeUpdate is called when an MsgAppResp arrives from the follower, with the // index acked by it. The method returns false if the given n index comes from // an outdated message. Otherwise it updates the progress and returns true.