diff --git a/raft.go b/raft.go index a2e5ad0f..1b52a163 100644 --- a/raft.go +++ b/raft.go @@ -827,7 +827,7 @@ func (r *raft) appendEntry(es ...pb.Entry) (accepted bool) { // if r.maybeCommit() { // r.bcastAppend() // } - r.send(pb.Message{To: r.id, Type: pb.MsgAppResp, Index: li}) + r.send(pb.Message{To: r.id, Type: pb.MsgAppResp, Index: li, Commit: r.raftLog.committed}) return true } @@ -1493,7 +1493,9 @@ func stepLeader(r *raft, m pb.Message) error { // equals pr.Match we know we don't m.Index+1 in our log, so moving // back to replicating state is not useful; besides pr.PendingSnapshot // would prevent it. - if pr.MaybeUpdate(m.Index) || (pr.Match == m.Index && pr.State == tracker.StateProbe) { + updated := pr.MaybeUpdate(m.Index) + pr.UpdateCommit(m.Commit) + if updated || (pr.Match == m.Index && pr.State == tracker.StateProbe) { switch { case pr.State == tracker.StateProbe: pr.BecomeReplicate() @@ -1765,12 +1767,12 @@ func (r *raft) handleAppendEntries(m pb.Message) { // message, and validate it before taking any action (e.g. bumping term). a := logSliceFromMsgApp(&m) - if a.prev.index < r.raftLog.committed { - r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: r.raftLog.committed}) + if commit := r.raftLog.committed; a.prev.index < commit { + r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: r.raftLog.committed, Commit: commit}) return } if mlastIndex, ok := r.raftLog.maybeAppend(a, m.Commit); ok { - r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: mlastIndex}) + r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: mlastIndex, Commit: r.raftLog.committed}) return } r.logger.Debugf("%x [logterm: %d, index: %d] rejected MsgApp [logterm: %d, index: %d] from %x", diff --git a/raft_paper_test.go b/raft_paper_test.go index eff31f63..5c72c280 100644 --- a/raft_paper_test.go +++ b/raft_paper_test.go @@ -28,6 +28,7 @@ package raft import ( "fmt" + "github.com/stretchr/testify/require" "reflect" "sort" "testing" @@ -606,20 +607,21 @@ func TestFollowerCheckMsgApp(t *testing.T) { term uint64 index uint64 windex uint64 + wcommit uint64 wreject bool wrejectHint uint64 wlogterm uint64 }{ // match with committed entries - {0, 0, 1, false, 0, 0}, - {ents[0].Term, ents[0].Index, 1, false, 0, 0}, + {0, 0, 1, 1, false, 0, 0}, + {ents[0].Term, ents[0].Index, 1, 1, false, 0, 0}, // match with uncommitted entries - {ents[1].Term, ents[1].Index, 2, false, 0, 0}, + {ents[1].Term, ents[1].Index, 2, 1, false, 0, 0}, // unmatch with existing entry - {ents[0].Term, ents[1].Index, ents[1].Index, true, 1, 1}, + {ents[0].Term, ents[1].Index, ents[1].Index, 0, true, 1, 1}, // unexisting entry - {ents[1].Term + 1, ents[1].Index + 1, ents[1].Index + 1, true, 2, 2}, + {ents[1].Term + 1, ents[1].Index + 1, ents[1].Index + 1, 0, true, 2, 2}, } for i, tt := range tests { storage := newTestMemoryStorage(withPeers(1, 2, 3)) @@ -630,13 +632,11 @@ func TestFollowerCheckMsgApp(t *testing.T) { r.Step(pb.Message{From: 2, To: 1, Type: pb.MsgApp, Term: 2, LogTerm: tt.term, Index: tt.index}) - msgs := r.readMessages() - wmsgs := []pb.Message{ - {From: 1, To: 2, Type: pb.MsgAppResp, Term: 2, Index: tt.windex, Reject: tt.wreject, RejectHint: tt.wrejectHint, LogTerm: tt.wlogterm}, - } - if !reflect.DeepEqual(msgs, wmsgs) { - t.Errorf("#%d: msgs = %+v, want %+v", i, msgs, wmsgs) + wmsg := pb.Message{ + From: 1, To: 2, Type: pb.MsgAppResp, Term: 2, Index: tt.windex, Commit: tt.wcommit, + Reject: tt.wreject, RejectHint: tt.wrejectHint, LogTerm: tt.wlogterm, } + require.Equal(t, []pb.Message{wmsg}, r.readMessages(), "#%d", i) } } diff --git a/testdata/async_storage_writes.txt b/testdata/async_storage_writes.txt index 02e59e5a..9434e6c0 100644 --- a/testdata/async_storage_writes.txt +++ b/testdata/async_storage_writes.txt @@ -90,7 +90,7 @@ stabilize 1->2 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] 1->3 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ - 1->1 MsgAppResp Term:1 Log:0/11 + 1->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 ] > 2 receiving messages @@ -101,7 +101,7 @@ stabilize Processing: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses: - 1->1 MsgAppResp Term:1 Log:0/11 + 1->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 > 2 handling Ready Ready MustSync=true: @@ -110,7 +110,7 @@ stabilize 1/11 EntryNormal "" Messages: 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ - 2->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 ] > 3 handling Ready @@ -120,27 +120,27 @@ stabilize 1/11 EntryNormal "" Messages: 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ - 3->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11 ] > 1 receiving messages - 1->1 MsgAppResp Term:1 Log:0/11 + 1->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses: - 2->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses: - 3->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:10 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/11 - 3->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:10 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:10 > 2 receiving messages AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 > 3 receiving messages @@ -177,7 +177,7 @@ stabilize 1/11 EntryNormal "" Messages: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses:[ - 2->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:11 ] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] @@ -189,7 +189,7 @@ stabilize 1/11 EntryNormal "" Messages: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses:[ - 3->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:11 ] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] @@ -200,12 +200,12 @@ stabilize Processing: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses: - 2->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:11 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses: - 3->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:11 > 2 processing apply thread Processing: 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] @@ -217,8 +217,8 @@ stabilize Responses: ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/11 - 3->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:11 + 3->1 MsgAppResp Term:1 Log:0/11 Commit:11 > 2 receiving messages ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] > 3 receiving messages @@ -238,7 +238,7 @@ process-ready 1 2 3 1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] 1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ - 1->1 MsgAppResp Term:1 Log:0/12 + 1->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 ] > 2 handling Ready @@ -261,7 +261,7 @@ process-ready 1 2 3 1/12 EntryNormal "prop_1" Messages: 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ - 2->1 MsgAppResp Term:1 Log:0/12 + 2->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 ] > 3 handling Ready @@ -270,7 +270,7 @@ process-ready 1 2 3 1/12 EntryNormal "prop_1" Messages: 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ - 3->1 MsgAppResp Term:1 Log:0/12 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12 ] @@ -288,7 +288,7 @@ process-ready 1 2 3 1->2 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_2"] 1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_2"] 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ - 1->1 MsgAppResp Term:1 Log:0/13 + 1->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13 ] > 2 handling Ready @@ -311,7 +311,7 @@ process-ready 1 2 3 1/13 EntryNormal "prop_2" Messages: 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ - 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13 ] > 3 handling Ready @@ -320,7 +320,7 @@ process-ready 1 2 3 1/13 EntryNormal "prop_2" Messages: 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ - 3->1 MsgAppResp Term:1 Log:0/13 + 3->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13 ] @@ -330,27 +330,27 @@ process-append-thread 1 2 3 Processing: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses: - 1->1 MsgAppResp Term:1 Log:0/12 + 1->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses: - 2->1 MsgAppResp Term:1 Log:0/12 + 2->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses: - 3->1 MsgAppResp Term:1 Log:0/12 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12 deliver-msgs 1 2 3 ---- -1->1 MsgAppResp Term:1 Log:0/12 +1->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 -2->1 MsgAppResp Term:1 Log:0/12 -3->1 MsgAppResp Term:1 Log:0/12 +2->1 MsgAppResp Term:1 Log:0/12 Commit:11 +3->1 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12 @@ -373,7 +373,7 @@ process-ready 1 2 3 1->2 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"] 1->3 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"] 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[ - 1->1 MsgAppResp Term:1 Log:0/14 + 1->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14 ] 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ @@ -404,8 +404,8 @@ process-ready 1 2 3 1/12 EntryNormal "prop_1" Messages: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[ - 2->1 MsgAppResp Term:1 Log:0/13 - 2->1 MsgAppResp Term:1 Log:0/14 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:12 + 2->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14 ] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ @@ -420,8 +420,8 @@ process-ready 1 2 3 1/12 EntryNormal "prop_1" Messages: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[ - 3->1 MsgAppResp Term:1 Log:0/13 - 3->1 MsgAppResp Term:1 Log:0/14 + 3->1 MsgAppResp Term:1 Log:0/13 Commit:12 + 3->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14 ] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ @@ -434,27 +434,27 @@ process-append-thread 1 2 3 Processing: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses: - 1->1 MsgAppResp Term:1 Log:0/13 + 1->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13 > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses: - 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses: - 3->1 MsgAppResp Term:1 Log:0/13 + 3->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13 deliver-msgs 1 2 3 ---- -1->1 MsgAppResp Term:1 Log:0/13 +1->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13 -2->1 MsgAppResp Term:1 Log:0/13 -3->1 MsgAppResp Term:1 Log:0/13 +2->1 MsgAppResp Term:1 Log:0/13 Commit:11 +3->1 MsgAppResp Term:1 Log:0/13 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13 @@ -477,7 +477,7 @@ process-ready 1 2 3 1->2 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"] 1->3 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"] 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[ - 1->1 MsgAppResp Term:1 Log:0/15 + 1->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 ] 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ @@ -508,8 +508,8 @@ process-ready 1 2 3 1/13 EntryNormal "prop_2" Messages: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[ - 2->1 MsgAppResp Term:1 Log:0/14 - 2->1 MsgAppResp Term:1 Log:0/15 + 2->1 MsgAppResp Term:1 Log:0/14 Commit:13 + 2->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 ] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ @@ -524,8 +524,8 @@ process-ready 1 2 3 1/13 EntryNormal "prop_2" Messages: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[ - 3->1 MsgAppResp Term:1 Log:0/14 - 3->1 MsgAppResp Term:1 Log:0/15 + 3->1 MsgAppResp Term:1 Log:0/14 Commit:13 + 3->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 ] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ @@ -538,21 +538,21 @@ process-append-thread 1 2 3 Processing: 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses: - 1->1 MsgAppResp Term:1 Log:0/14 + 1->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14 > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses: - 2->1 MsgAppResp Term:1 Log:0/13 - 2->1 MsgAppResp Term:1 Log:0/14 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:12 + 2->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses: - 3->1 MsgAppResp Term:1 Log:0/13 - 3->1 MsgAppResp Term:1 Log:0/14 + 3->1 MsgAppResp Term:1 Log:0/13 Commit:12 + 3->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14 process-apply-thread 1 2 3 @@ -575,12 +575,12 @@ process-apply-thread 1 2 3 deliver-msgs 1 2 3 ---- -1->1 MsgAppResp Term:1 Log:0/14 +1->1 MsgAppResp Term:1 Log:0/14 Commit:12 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14 -2->1 MsgAppResp Term:1 Log:0/13 -2->1 MsgAppResp Term:1 Log:0/14 -3->1 MsgAppResp Term:1 Log:0/13 -3->1 MsgAppResp Term:1 Log:0/14 +2->1 MsgAppResp Term:1 Log:0/13 Commit:12 +2->1 MsgAppResp Term:1 Log:0/14 Commit:12 +3->1 MsgAppResp Term:1 Log:0/13 Commit:12 +3->1 MsgAppResp Term:1 Log:0/14 Commit:12 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14 ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] @@ -624,7 +624,7 @@ process-ready 1 2 3 1/14 EntryNormal "prop_3" Messages: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[ - 2->1 MsgAppResp Term:1 Log:0/15 + 2->1 MsgAppResp Term:1 Log:0/15 Commit:14 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 ] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ @@ -637,7 +637,7 @@ process-ready 1 2 3 1/14 EntryNormal "prop_3" Messages: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[ - 3->1 MsgAppResp Term:1 Log:0/15 + 3->1 MsgAppResp Term:1 Log:0/15 Commit:14 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 ] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ @@ -650,21 +650,21 @@ process-append-thread 1 2 3 Processing: 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses: - 1->1 MsgAppResp Term:1 Log:0/15 + 1->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses: - 2->1 MsgAppResp Term:1 Log:0/14 - 2->1 MsgAppResp Term:1 Log:0/15 + 2->1 MsgAppResp Term:1 Log:0/14 Commit:13 + 2->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses: - 3->1 MsgAppResp Term:1 Log:0/14 - 3->1 MsgAppResp Term:1 Log:0/15 + 3->1 MsgAppResp Term:1 Log:0/14 Commit:13 + 3->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 process-apply-thread 1 2 3 @@ -687,12 +687,12 @@ process-apply-thread 1 2 3 deliver-msgs 1 2 3 ---- -1->1 MsgAppResp Term:1 Log:0/15 +1->1 MsgAppResp Term:1 Log:0/15 Commit:13 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 -2->1 MsgAppResp Term:1 Log:0/14 -2->1 MsgAppResp Term:1 Log:0/15 -3->1 MsgAppResp Term:1 Log:0/14 -3->1 MsgAppResp Term:1 Log:0/15 +2->1 MsgAppResp Term:1 Log:0/14 Commit:13 +2->1 MsgAppResp Term:1 Log:0/15 Commit:13 +3->1 MsgAppResp Term:1 Log:0/14 Commit:13 +3->1 MsgAppResp Term:1 Log:0/15 Commit:13 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] @@ -734,7 +734,7 @@ process-ready 1 2 3 1/15 EntryNormal "prop_4" Messages: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[ - 2->1 MsgAppResp Term:1 Log:0/15 + 2->1 MsgAppResp Term:1 Log:0/15 Commit:15 ] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] @@ -746,7 +746,7 @@ process-ready 1 2 3 1/15 EntryNormal "prop_4" Messages: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[ - 3->1 MsgAppResp Term:1 Log:0/15 + 3->1 MsgAppResp Term:1 Log:0/15 Commit:15 ] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] @@ -758,13 +758,13 @@ process-append-thread 2 3 Processing: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses: - 2->1 MsgAppResp Term:1 Log:0/15 + 2->1 MsgAppResp Term:1 Log:0/15 Commit:14 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses: - 3->1 MsgAppResp Term:1 Log:0/15 + 3->1 MsgAppResp Term:1 Log:0/15 Commit:14 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 process-apply-thread 1 2 3 @@ -787,8 +787,8 @@ process-apply-thread 1 2 3 deliver-msgs 1 2 3 ---- -2->1 MsgAppResp Term:1 Log:0/15 -3->1 MsgAppResp Term:1 Log:0/15 +2->1 MsgAppResp Term:1 Log:0/15 Commit:14 +3->1 MsgAppResp Term:1 Log:0/15 Commit:14 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 INFO entry at index 15 missing from unstable log; ignoring @@ -812,12 +812,12 @@ process-append-thread 2 3 Processing: 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses: - 2->1 MsgAppResp Term:1 Log:0/15 + 2->1 MsgAppResp Term:1 Log:0/15 Commit:15 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses: - 3->1 MsgAppResp Term:1 Log:0/15 + 3->1 MsgAppResp Term:1 Log:0/15 Commit:15 process-apply-thread 1 2 3 ---- @@ -839,8 +839,8 @@ process-apply-thread 1 2 3 deliver-msgs 1 2 3 ---- -2->1 MsgAppResp Term:1 Log:0/15 -3->1 MsgAppResp Term:1 Log:0/15 +2->1 MsgAppResp Term:1 Log:0/15 Commit:15 +3->1 MsgAppResp Term:1 Log:0/15 Commit:15 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] diff --git a/testdata/async_storage_writes_append_aba_race.txt b/testdata/async_storage_writes_append_aba_race.txt index 83964fe7..f2cbfb02 100644 --- a/testdata/async_storage_writes_append_aba_race.txt +++ b/testdata/async_storage_writes_append_aba_race.txt @@ -43,7 +43,7 @@ Messages: 2->6 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] 2->7 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[ - 2->2 MsgAppResp Term:1 Log:0/12 + 2->2 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 ] @@ -65,7 +65,7 @@ Entries: 1/12 EntryNormal "init_prop" Messages: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[ - 1->2 MsgAppResp Term:1 Log:0/12 + 1->2 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 ] @@ -198,7 +198,7 @@ Messages: 3->6 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] 3->7 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[2/12 EntryNormal ""] Responses:[ - 3->3 MsgAppResp Term:2 Log:0/12 + 3->3 MsgAppResp Term:2 Log:0/12 Commit:11 AppendThread->3 MsgStorageAppendResp Term:2 Log:2/12 ] @@ -229,7 +229,7 @@ Entries: Messages: 1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] Responses:[ 1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0) - 1->3 MsgAppResp Term:2 Log:0/12 + 1->3 MsgAppResp Term:2 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12 ] @@ -355,7 +355,7 @@ Messages: 4->6 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] 4->7 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] 4->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[ - 4->4 MsgAppResp Term:3 Log:0/12 + 4->4 MsgAppResp Term:3 Log:0/12 Commit:11 AppendThread->4 MsgStorageAppendResp Term:3 Log:3/12 ] @@ -425,7 +425,7 @@ Entries: 3/12 EntryNormal "" Messages: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[ - 1->4 MsgAppResp Term:3 Log:0/12 + 1->4 MsgAppResp Term:3 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12 ] @@ -442,7 +442,7 @@ process-append-thread 1 Processing: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses: -1->2 MsgAppResp Term:1 Log:0/12 +1->2 MsgAppResp Term:1 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 raft-log 1 @@ -469,7 +469,7 @@ Processing: 1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] Responses: 1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0) -1->3 MsgAppResp Term:2 Log:0/12 +1->3 MsgAppResp Term:2 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12 raft-log 1 @@ -504,7 +504,7 @@ process-append-thread 1 Processing: 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses: -1->4 MsgAppResp Term:3 Log:0/12 +1->4 MsgAppResp Term:3 Log:0/12 Commit:11 AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12 raft-log 1 diff --git a/testdata/campaign.txt b/testdata/campaign.txt index 4eefab36..3538aa64 100644 --- a/testdata/campaign.txt +++ b/testdata/campaign.txt @@ -76,17 +76,17 @@ stabilize Entries: 1/3 EntryNormal "" Messages: - 2->1 MsgAppResp Term:1 Log:0/3 + 2->1 MsgAppResp Term:1 Log:0/3 Commit:2 > 3 handling Ready Ready MustSync=true: Lead:1 State:StateFollower Entries: 1/3 EntryNormal "" Messages: - 3->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgAppResp Term:1 Log:0/3 Commit:2 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/3 - 3->1 MsgAppResp Term:1 Log:0/3 + 2->1 MsgAppResp Term:1 Log:0/3 Commit:2 + 3->1 MsgAppResp Term:1 Log:0/3 Commit:2 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:3 @@ -105,14 +105,14 @@ stabilize CommittedEntries: 1/3 EntryNormal "" Messages: - 2->1 MsgAppResp Term:1 Log:0/3 + 2->1 MsgAppResp Term:1 Log:0/3 Commit:3 > 3 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:3 CommittedEntries: 1/3 EntryNormal "" Messages: - 3->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgAppResp Term:1 Log:0/3 Commit:3 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/3 - 3->1 MsgAppResp Term:1 Log:0/3 + 2->1 MsgAppResp Term:1 Log:0/3 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/3 Commit:3 diff --git a/testdata/campaign_learner_must_vote.txt b/testdata/campaign_learner_must_vote.txt index d05de374..9da576ff 100644 --- a/testdata/campaign_learner_must_vote.txt +++ b/testdata/campaign_learner_must_vote.txt @@ -135,10 +135,10 @@ stabilize 2 3 CommittedEntries: 1/4 EntryConfChangeV2 v3 Messages: - 3->2 MsgAppResp Term:2 Log:0/5 + 3->2 MsgAppResp Term:2 Log:0/5 Commit:4 INFO 3 switched to configuration voters=(1 2 3) > 2 receiving messages - 3->2 MsgAppResp Term:2 Log:0/5 + 3->2 MsgAppResp Term:2 Log:0/5 Commit:4 > 2 handling Ready Ready MustSync=false: HardState Term:2 Vote:2 Commit:5 @@ -154,6 +154,6 @@ stabilize 2 3 CommittedEntries: 2/5 EntryNormal "" Messages: - 3->2 MsgAppResp Term:2 Log:0/5 + 3->2 MsgAppResp Term:2 Log:0/5 Commit:5 > 2 receiving messages - 3->2 MsgAppResp Term:2 Log:0/5 + 3->2 MsgAppResp Term:2 Log:0/5 Commit:5 diff --git a/testdata/checkquorum.txt b/testdata/checkquorum.txt index b25c1e63..065e0754 100644 --- a/testdata/checkquorum.txt +++ b/testdata/checkquorum.txt @@ -193,7 +193,7 @@ stabilize Entries: 3/12 EntryNormal "" Messages: - 1->2 MsgAppResp Term:3 Log:0/12 + 1->2 MsgAppResp Term:3 Log:0/12 Commit:11 > 3 handling Ready Ready MustSync=true: Lead:2 State:StateFollower @@ -201,10 +201,10 @@ stabilize Entries: 3/12 EntryNormal "" Messages: - 3->2 MsgAppResp Term:3 Log:0/12 + 3->2 MsgAppResp Term:3 Log:0/12 Commit:11 > 2 receiving messages - 1->2 MsgAppResp Term:3 Log:0/12 - 3->2 MsgAppResp Term:3 Log:0/12 + 1->2 MsgAppResp Term:3 Log:0/12 Commit:11 + 3->2 MsgAppResp Term:3 Log:0/12 Commit:11 > 2 handling Ready Ready MustSync=false: HardState Term:3 Vote:2 Commit:12 @@ -223,14 +223,14 @@ stabilize CommittedEntries: 3/12 EntryNormal "" Messages: - 1->2 MsgAppResp Term:3 Log:0/12 + 1->2 MsgAppResp Term:3 Log:0/12 Commit:12 > 3 handling Ready Ready MustSync=false: HardState Term:3 Commit:12 CommittedEntries: 3/12 EntryNormal "" Messages: - 3->2 MsgAppResp Term:3 Log:0/12 + 3->2 MsgAppResp Term:3 Log:0/12 Commit:12 > 2 receiving messages - 1->2 MsgAppResp Term:3 Log:0/12 - 3->2 MsgAppResp Term:3 Log:0/12 + 1->2 MsgAppResp Term:3 Log:0/12 Commit:12 + 3->2 MsgAppResp Term:3 Log:0/12 Commit:12 diff --git a/testdata/confchange_v1_remove_leader.txt b/testdata/confchange_v1_remove_leader.txt index cc91508a..fde99c82 100644 --- a/testdata/confchange_v1_remove_leader.txt +++ b/testdata/confchange_v1_remove_leader.txt @@ -70,8 +70,8 @@ stabilize 2 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 2->1 MsgAppResp Term:1 Log:0/4 - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:3 # Put another entry in n1's log. propose 1 bar @@ -92,8 +92,8 @@ stabilize 1 1->2 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] 1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:3 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:5 @@ -129,18 +129,18 @@ stabilize 2 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 INFO 2 switched to configuration voters=(2 3) # ... which thankfully is what we see on the leader. stabilize 1 ---- > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 # When n3 responds, quorum is reached and everything falls into place. stabilize @@ -162,18 +162,18 @@ stabilize 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 3->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 INFO 3 switched to configuration voters=(2 3) > 1 receiving messages - 3->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:6 @@ -192,17 +192,17 @@ stabilize CommittedEntries: 1/6 EntryNormal "bar" Messages: - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:6 > 3 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:6 CommittedEntries: 1/6 EntryNormal "bar" Messages: - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:6 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:6 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:6 # However not all is well. n1 is still leader but unconditionally drops all # proposals on the floor, so we're effectively stuck if it still heartbeats diff --git a/testdata/confchange_v1_remove_leader_stepdown.txt b/testdata/confchange_v1_remove_leader_stepdown.txt index fe397650..b9e14ff0 100644 --- a/testdata/confchange_v1_remove_leader_stepdown.txt +++ b/testdata/confchange_v1_remove_leader_stepdown.txt @@ -71,8 +71,8 @@ stabilize 2 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 2->1 MsgAppResp Term:1 Log:0/4 - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:3 # Put another entry in n1's log. propose 1 bar @@ -91,8 +91,8 @@ stabilize 1 1->2 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] 1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:3 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:5 @@ -132,18 +132,18 @@ stabilize 2 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 INFO 2 switched to configuration voters=(2 3) # ...because the old leader n1 ignores the append responses. stabilize 1 ---- > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 # When n3 responds, quorum is reached and everything falls into place. stabilize @@ -165,18 +165,18 @@ stabilize 1/4 EntryConfChange r1 1/5 EntryNormal "foo" Messages: - 3->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 INFO 3 switched to configuration voters=(2 3) > 1 receiving messages - 3->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 # n1 can no longer propose. propose 1 baz diff --git a/testdata/confchange_v2_add_double_auto.txt b/testdata/confchange_v2_add_double_auto.txt index f290c980..0d86ac1a 100644 --- a/testdata/confchange_v2_add_double_auto.txt +++ b/testdata/confchange_v2_add_double_auto.txt @@ -128,9 +128,9 @@ stabilize 1 2 Entries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:5 @@ -147,10 +147,10 @@ stabilize 1 2 CommittedEntries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 INFO 2 switched to configuration voters=(1 2 3) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 # n3 immediately receives a snapshot in the final configuration. stabilize 1 3 @@ -230,13 +230,13 @@ stabilize 2 3 Entries: 1/6 EntryConfChangeV2 r2 r3 Messages: - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 > 3 handling Ready Ready MustSync=true: Entries: 1/6 EntryConfChangeV2 r2 r3 Messages: - 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 # n1 gets some more proposals. This is part of a regression test: There used to # be a bug in which these proposals would prompt the leader to transition out of @@ -264,8 +264,8 @@ stabilize 1 1->2 MsgApp Term:1 Log:1/7 Commit:5 Entries:[1/8 EntryNormal "bar"] 1->3 MsgApp Term:1 Log:1/7 Commit:5 Entries:[1/8 EntryNormal "bar"] > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/6 - 3->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/6 Commit:5 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:6 @@ -307,10 +307,10 @@ stabilize 2 3 CommittedEntries: 1/6 EntryConfChangeV2 r2 r3 Messages: - 2->1 MsgAppResp Term:1 Log:0/7 - 2->1 MsgAppResp Term:1 Log:0/8 - 2->1 MsgAppResp Term:1 Log:0/8 - 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/8 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/8 Commit:6 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:6 INFO 2 switched to configuration voters=(1)&&(1 2 3) autoleave > 3 handling Ready Ready MustSync=true: @@ -322,10 +322,10 @@ stabilize 2 3 CommittedEntries: 1/6 EntryConfChangeV2 r2 r3 Messages: - 3->1 MsgAppResp Term:1 Log:0/7 - 3->1 MsgAppResp Term:1 Log:0/8 - 3->1 MsgAppResp Term:1 Log:0/8 - 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/7 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/8 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/8 Commit:6 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:6 INFO 3 switched to configuration voters=(1)&&(1 2 3) autoleave # n2 and n3 also leave the joint config and the dust settles. We see at the very @@ -334,14 +334,14 @@ stabilize 2 3 stabilize ---- > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/7 - 2->1 MsgAppResp Term:1 Log:0/8 - 2->1 MsgAppResp Term:1 Log:0/8 - 2->1 MsgAppResp Term:1 Log:0/9 - 3->1 MsgAppResp Term:1 Log:0/7 - 3->1 MsgAppResp Term:1 Log:0/8 - 3->1 MsgAppResp Term:1 Log:0/8 - 3->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/8 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/8 Commit:6 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:6 + 3->1 MsgAppResp Term:1 Log:0/7 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/8 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/8 Commit:6 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:6 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:9 @@ -373,9 +373,9 @@ stabilize 1/8 EntryNormal "bar" 1/9 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/9 - 2->1 MsgAppResp Term:1 Log:0/9 - 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:7 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:8 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:9 INFO 2 switched to configuration voters=(1) > 3 handling Ready Ready MustSync=false: @@ -385,20 +385,20 @@ stabilize 1/8 EntryNormal "bar" 1/9 EntryConfChangeV2 Messages: - 3->1 MsgAppResp Term:1 Log:0/9 - 3->1 MsgAppResp Term:1 Log:0/9 - 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:7 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:8 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:9 INFO 3 switched to configuration voters=(1) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:7 raft: cannot step as peer not found - 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:8 raft: cannot step as peer not found - 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/9 Commit:9 raft: cannot step as peer not found - 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:7 raft: cannot step as peer not found - 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:8 raft: cannot step as peer not found - 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/9 Commit:9 raft: cannot step as peer not found diff --git a/testdata/confchange_v2_add_double_implicit.txt b/testdata/confchange_v2_add_double_implicit.txt index 2db3b7c1..30f26f68 100644 --- a/testdata/confchange_v2_add_double_implicit.txt +++ b/testdata/confchange_v2_add_double_implicit.txt @@ -111,9 +111,9 @@ stabilize 1 2 Entries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:5 @@ -130,7 +130,7 @@ stabilize 1 2 CommittedEntries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 INFO 2 switched to configuration voters=(1 2) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 diff --git a/testdata/confchange_v2_add_single_explicit.txt b/testdata/confchange_v2_add_single_explicit.txt index 1d390e55..770c9a27 100644 --- a/testdata/confchange_v2_add_single_explicit.txt +++ b/testdata/confchange_v2_add_single_explicit.txt @@ -129,11 +129,11 @@ stabilize 1/5 EntryNormal "" 1/6 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:4 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:6 @@ -154,12 +154,12 @@ stabilize 1/5 EntryNormal "" 1/6 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:6 INFO 2 switched to configuration voters=(1 2) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/6 - 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:5 + 2->1 MsgAppResp Term:1 Log:0/6 Commit:6 # Check that trying to transition out again won't do anything. propose-conf-change 1 @@ -182,9 +182,9 @@ stabilize Entries: 1/7 EntryNormal "" Messages: - 2->1 MsgAppResp Term:1 Log:0/7 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:6 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/7 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:6 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:7 @@ -200,6 +200,6 @@ stabilize CommittedEntries: 1/7 EntryNormal "" Messages: - 2->1 MsgAppResp Term:1 Log:0/7 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:7 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/7 + 2->1 MsgAppResp Term:1 Log:0/7 Commit:7 diff --git a/testdata/confchange_v2_replace_leader.txt b/testdata/confchange_v2_replace_leader.txt index 628cc447..a2b8425c 100644 --- a/testdata/confchange_v2_replace_leader.txt +++ b/testdata/confchange_v2_replace_leader.txt @@ -65,16 +65,16 @@ stabilize Entries: 1/4 EntryConfChangeV2 r1 v4 Messages: - 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 > 3 handling Ready Ready MustSync=true: Entries: 1/4 EntryConfChangeV2 r1 v4 Messages: - 3->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:3 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:3 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:4 @@ -98,7 +98,7 @@ stabilize CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: - 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 INFO 2 switched to configuration voters=(2 3 4)&&(1 2 3) > 3 handling Ready Ready MustSync=false: @@ -106,11 +106,11 @@ stabilize CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: - 3->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:4 INFO 3 switched to configuration voters=(2 3 4)&&(1 2 3) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/4 - 3->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 4 receiving messages 1->4 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 r1 v4] INFO 4 [term: 0] received a MsgApp message with higher term from 1 [term: 1] @@ -247,25 +247,25 @@ stabilize Entries: 2/5 EntryNormal "" Messages: - 1->4 MsgAppResp Term:2 Log:0/5 + 1->4 MsgAppResp Term:2 Log:0/5 Commit:4 > 2 handling Ready Ready MustSync=true: Lead:4 State:StateFollower Entries: 2/5 EntryNormal "" Messages: - 2->4 MsgAppResp Term:2 Log:0/5 + 2->4 MsgAppResp Term:2 Log:0/5 Commit:4 > 3 handling Ready Ready MustSync=true: Lead:4 State:StateFollower Entries: 2/5 EntryNormal "" Messages: - 3->4 MsgAppResp Term:2 Log:0/5 + 3->4 MsgAppResp Term:2 Log:0/5 Commit:4 > 4 receiving messages - 1->4 MsgAppResp Term:2 Log:0/5 - 2->4 MsgAppResp Term:2 Log:0/5 - 3->4 MsgAppResp Term:2 Log:0/5 + 1->4 MsgAppResp Term:2 Log:0/5 Commit:4 + 2->4 MsgAppResp Term:2 Log:0/5 Commit:4 + 3->4 MsgAppResp Term:2 Log:0/5 Commit:4 > 4 handling Ready Ready MustSync=false: HardState Term:2 Vote:4 Commit:5 @@ -287,25 +287,25 @@ stabilize CommittedEntries: 2/5 EntryNormal "" Messages: - 1->4 MsgAppResp Term:2 Log:0/5 + 1->4 MsgAppResp Term:2 Log:0/5 Commit:5 > 2 handling Ready Ready MustSync=false: HardState Term:2 Vote:4 Commit:5 CommittedEntries: 2/5 EntryNormal "" Messages: - 2->4 MsgAppResp Term:2 Log:0/5 + 2->4 MsgAppResp Term:2 Log:0/5 Commit:5 > 3 handling Ready Ready MustSync=false: HardState Term:2 Vote:4 Commit:5 CommittedEntries: 2/5 EntryNormal "" Messages: - 3->4 MsgAppResp Term:2 Log:0/5 + 3->4 MsgAppResp Term:2 Log:0/5 Commit:5 > 4 receiving messages - 1->4 MsgAppResp Term:2 Log:0/5 - 2->4 MsgAppResp Term:2 Log:0/5 - 3->4 MsgAppResp Term:2 Log:0/5 + 1->4 MsgAppResp Term:2 Log:0/5 Commit:5 + 2->4 MsgAppResp Term:2 Log:0/5 Commit:5 + 3->4 MsgAppResp Term:2 Log:0/5 Commit:5 # Leadership transfer succeeded. raft-state @@ -342,23 +342,23 @@ stabilize Entries: 2/6 EntryConfChangeV2 Messages: - 1->4 MsgAppResp Term:2 Log:0/6 + 1->4 MsgAppResp Term:2 Log:0/6 Commit:5 > 2 handling Ready Ready MustSync=true: Entries: 2/6 EntryConfChangeV2 Messages: - 2->4 MsgAppResp Term:2 Log:0/6 + 2->4 MsgAppResp Term:2 Log:0/6 Commit:5 > 3 handling Ready Ready MustSync=true: Entries: 2/6 EntryConfChangeV2 Messages: - 3->4 MsgAppResp Term:2 Log:0/6 + 3->4 MsgAppResp Term:2 Log:0/6 Commit:5 > 4 receiving messages - 1->4 MsgAppResp Term:2 Log:0/6 - 2->4 MsgAppResp Term:2 Log:0/6 - 3->4 MsgAppResp Term:2 Log:0/6 + 1->4 MsgAppResp Term:2 Log:0/6 Commit:5 + 2->4 MsgAppResp Term:2 Log:0/6 Commit:5 + 3->4 MsgAppResp Term:2 Log:0/6 Commit:5 > 4 handling Ready Ready MustSync=false: HardState Term:2 Vote:4 Commit:6 @@ -381,7 +381,7 @@ stabilize CommittedEntries: 2/6 EntryConfChangeV2 Messages: - 1->4 MsgAppResp Term:2 Log:0/6 + 1->4 MsgAppResp Term:2 Log:0/6 Commit:6 INFO 1 switched to configuration voters=(2 3 4) > 2 handling Ready Ready MustSync=false: @@ -389,7 +389,7 @@ stabilize CommittedEntries: 2/6 EntryConfChangeV2 Messages: - 2->4 MsgAppResp Term:2 Log:0/6 + 2->4 MsgAppResp Term:2 Log:0/6 Commit:6 INFO 2 switched to configuration voters=(2 3 4) > 3 handling Ready Ready MustSync=false: @@ -397,13 +397,13 @@ stabilize CommittedEntries: 2/6 EntryConfChangeV2 Messages: - 3->4 MsgAppResp Term:2 Log:0/6 + 3->4 MsgAppResp Term:2 Log:0/6 Commit:6 INFO 3 switched to configuration voters=(2 3 4) > 4 receiving messages - 1->4 MsgAppResp Term:2 Log:0/6 + 1->4 MsgAppResp Term:2 Log:0/6 Commit:6 raft: cannot step as peer not found - 2->4 MsgAppResp Term:2 Log:0/6 - 3->4 MsgAppResp Term:2 Log:0/6 + 2->4 MsgAppResp Term:2 Log:0/6 Commit:6 + 3->4 MsgAppResp Term:2 Log:0/6 Commit:6 # n1 is out of the configuration. raft-state diff --git a/testdata/confchange_v2_replace_leader_stepdown.txt b/testdata/confchange_v2_replace_leader_stepdown.txt index 62d01d23..ecad1d7c 100644 --- a/testdata/confchange_v2_replace_leader_stepdown.txt +++ b/testdata/confchange_v2_replace_leader_stepdown.txt @@ -93,23 +93,23 @@ stabilize Entries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 3 handling Ready Ready MustSync=true: Entries: 1/5 EntryConfChangeV2 Messages: - 3->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 4 handling Ready Ready MustSync=true: Entries: 1/5 EntryConfChangeV2 Messages: - 4->1 MsgAppResp Term:1 Log:0/5 + 4->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/5 - 4->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:4 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:4 + 4->1 MsgAppResp Term:1 Log:0/5 Commit:4 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:5 @@ -136,7 +136,7 @@ stabilize CommittedEntries: 1/5 EntryConfChangeV2 Messages: - 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 INFO 2 switched to configuration voters=(2 3 4) > 3 handling Ready Ready MustSync=false: @@ -144,7 +144,7 @@ stabilize CommittedEntries: 1/5 EntryConfChangeV2 Messages: - 3->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:5 INFO 3 switched to configuration voters=(2 3 4) > 4 handling Ready Ready MustSync=false: @@ -152,12 +152,12 @@ stabilize CommittedEntries: 1/5 EntryConfChangeV2 Messages: - 4->1 MsgAppResp Term:1 Log:0/5 + 4->1 MsgAppResp Term:1 Log:0/5 Commit:5 INFO 4 switched to configuration voters=(2 3 4) > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/5 - 3->1 MsgAppResp Term:1 Log:0/5 - 4->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/5 Commit:5 + 3->1 MsgAppResp Term:1 Log:0/5 Commit:5 + 4->1 MsgAppResp Term:1 Log:0/5 Commit:5 # n1 is out of the configuration. raft-state diff --git a/testdata/forget_leader_prevote_checkquorum.txt b/testdata/forget_leader_prevote_checkquorum.txt index 9b3b80ff..d204b9d7 100644 --- a/testdata/forget_leader_prevote_checkquorum.txt +++ b/testdata/forget_leader_prevote_checkquorum.txt @@ -182,7 +182,7 @@ stabilize 2 Entries: 2/13 EntryNormal "prop_1" Messages: - 2->3 MsgAppResp Term:2 Log:0/13 + 2->3 MsgAppResp Term:2 Log:0/13 Commit:12 forget-leader 2 ---- diff --git a/testdata/heartbeat_resp_recovers_from_probing.txt b/testdata/heartbeat_resp_recovers_from_probing.txt index e606a155..bb6eef1e 100644 --- a/testdata/heartbeat_resp_recovers_from_probing.txt +++ b/testdata/heartbeat_resp_recovers_from_probing.txt @@ -79,9 +79,9 @@ stabilize > 2 handling Ready Ready MustSync=false: Messages: - 2->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:11 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/11 + 2->1 MsgAppResp Term:1 Log:0/11 Commit:11 status 1 ---- diff --git a/testdata/lagging_commit.txt b/testdata/lagging_commit.txt index 8f8ba336..551d4e75 100644 --- a/testdata/lagging_commit.txt +++ b/testdata/lagging_commit.txt @@ -51,15 +51,15 @@ Entries: 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" Messages: -3->1 MsgAppResp Term:1 Log:0/12 -3->1 MsgAppResp Term:1 Log:0/13 +3->1 MsgAppResp Term:1 Log:0/12 Commit:11 +3->1 MsgAppResp Term:1 Log:0/13 Commit:11 # Suppose there is a network blip which prevents the leader learning that the # follower 3 has appended the proposed entries to the log. deliver-msgs drop=(1) ---- -dropped: 3->1 MsgAppResp Term:1 Log:0/12 -dropped: 3->1 MsgAppResp Term:1 Log:0/13 +dropped: 3->1 MsgAppResp Term:1 Log:0/12 Commit:11 +dropped: 3->1 MsgAppResp Term:1 Log:0/13 Commit:11 # In the meantime, the entries are committed, and the leader sends the commit # index to all the followers. @@ -71,11 +71,11 @@ stabilize 1 2 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" Messages: - 2->1 MsgAppResp Term:1 Log:0/12 - 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/12 Commit:11 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:11 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/12 - 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/12 Commit:11 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:11 > 1 handling Ready Ready MustSync=false: HardState Term:1 Vote:1 Commit:13 @@ -97,11 +97,11 @@ stabilize 1 2 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" Messages: - 2->1 MsgAppResp Term:1 Log:0/13 - 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:12 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:13 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/13 - 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:12 + 2->1 MsgAppResp Term:1 Log:0/13 Commit:13 # The network blip prevents the follower 3 from learning that the previously # appended entries are now committed. @@ -169,6 +169,6 @@ stabilize 1 3 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" Messages: - 3->1 MsgAppResp Term:1 Log:0/13 + 3->1 MsgAppResp Term:1 Log:0/13 Commit:13 > 1 receiving messages - 3->1 MsgAppResp Term:1 Log:0/13 + 3->1 MsgAppResp Term:1 Log:0/13 Commit:13 diff --git a/testdata/prevote.txt b/testdata/prevote.txt index db763d35..f4926d43 100644 --- a/testdata/prevote.txt +++ b/testdata/prevote.txt @@ -49,7 +49,7 @@ Ready MustSync=true: Entries: 1/12 EntryNormal "prop_1" Messages: -2->1 MsgAppResp Term:1 Log:0/12 +2->1 MsgAppResp Term:1 Log:0/12 Commit:11 # 3 is now behind on its log. Attempt to campaign. raft-log 3 @@ -75,7 +75,7 @@ INFO 3 has received 1 MsgPreVoteResp votes and 0 vote rejections deliver-msgs 1 2 ---- -2->1 MsgAppResp Term:1 Log:0/12 +2->1 MsgAppResp Term:1 Log:0/12 Commit:11 3->1 MsgPreVote Term:2 Log:1/11 INFO 1 [logterm: 1, index: 12, vote: 1] rejected MsgPreVote from 3 [logterm: 1, index: 11] at term 1 3->2 MsgPreVote Term:2 Log:1/11 @@ -111,7 +111,7 @@ stabilize CommittedEntries: 1/12 EntryNormal "prop_1" Messages: - 2->1 MsgAppResp Term:1 Log:0/12 + 2->1 MsgAppResp Term:1 Log:0/12 Commit:12 > 3 handling Ready Ready MustSync=true: Lead:1 State:StateFollower @@ -121,12 +121,12 @@ stabilize CommittedEntries: 1/12 EntryNormal "prop_1" Messages: - 3->1 MsgAppResp Term:1 Log:0/12 - 3->1 MsgAppResp Term:1 Log:0/12 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:11 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:12 > 1 receiving messages - 2->1 MsgAppResp Term:1 Log:0/12 - 3->1 MsgAppResp Term:1 Log:0/12 - 3->1 MsgAppResp Term:1 Log:0/12 + 2->1 MsgAppResp Term:1 Log:0/12 Commit:12 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:11 + 3->1 MsgAppResp Term:1 Log:0/12 Commit:12 # Let 2 campaign. It should succeed, since it's up-to-date on the log. campaign 2 @@ -223,17 +223,17 @@ stabilize Entries: 2/13 EntryNormal "" Messages: - 1->2 MsgAppResp Term:2 Log:0/13 + 1->2 MsgAppResp Term:2 Log:0/13 Commit:12 > 3 handling Ready Ready MustSync=true: Lead:2 State:StateFollower Entries: 2/13 EntryNormal "" Messages: - 3->2 MsgAppResp Term:2 Log:0/13 + 3->2 MsgAppResp Term:2 Log:0/13 Commit:12 > 2 receiving messages - 1->2 MsgAppResp Term:2 Log:0/13 - 3->2 MsgAppResp Term:2 Log:0/13 + 1->2 MsgAppResp Term:2 Log:0/13 Commit:12 + 3->2 MsgAppResp Term:2 Log:0/13 Commit:12 > 2 handling Ready Ready MustSync=false: HardState Term:2 Vote:2 Commit:13 @@ -252,14 +252,14 @@ stabilize CommittedEntries: 2/13 EntryNormal "" Messages: - 1->2 MsgAppResp Term:2 Log:0/13 + 1->2 MsgAppResp Term:2 Log:0/13 Commit:13 > 3 handling Ready Ready MustSync=false: HardState Term:2 Vote:2 Commit:13 CommittedEntries: 2/13 EntryNormal "" Messages: - 3->2 MsgAppResp Term:2 Log:0/13 + 3->2 MsgAppResp Term:2 Log:0/13 Commit:13 > 2 receiving messages - 1->2 MsgAppResp Term:2 Log:0/13 - 3->2 MsgAppResp Term:2 Log:0/13 + 1->2 MsgAppResp Term:2 Log:0/13 Commit:13 + 3->2 MsgAppResp Term:2 Log:0/13 Commit:13 diff --git a/testdata/prevote_checkquorum.txt b/testdata/prevote_checkquorum.txt index 6db6662b..a7cc57ae 100644 --- a/testdata/prevote_checkquorum.txt +++ b/testdata/prevote_checkquorum.txt @@ -147,17 +147,17 @@ stabilize Entries: 2/12 EntryNormal "" Messages: - 1->3 MsgAppResp Term:2 Log:0/12 + 1->3 MsgAppResp Term:2 Log:0/12 Commit:11 > 2 handling Ready Ready MustSync=true: Lead:3 State:StateFollower Entries: 2/12 EntryNormal "" Messages: - 2->3 MsgAppResp Term:2 Log:0/12 + 2->3 MsgAppResp Term:2 Log:0/12 Commit:11 > 3 receiving messages - 1->3 MsgAppResp Term:2 Log:0/12 - 2->3 MsgAppResp Term:2 Log:0/12 + 1->3 MsgAppResp Term:2 Log:0/12 Commit:11 + 2->3 MsgAppResp Term:2 Log:0/12 Commit:11 > 3 handling Ready Ready MustSync=false: HardState Term:2 Vote:3 Commit:12 @@ -176,17 +176,17 @@ stabilize CommittedEntries: 2/12 EntryNormal "" Messages: - 1->3 MsgAppResp Term:2 Log:0/12 + 1->3 MsgAppResp Term:2 Log:0/12 Commit:12 > 2 handling Ready Ready MustSync=false: HardState Term:2 Vote:3 Commit:12 CommittedEntries: 2/12 EntryNormal "" Messages: - 2->3 MsgAppResp Term:2 Log:0/12 + 2->3 MsgAppResp Term:2 Log:0/12 Commit:12 > 3 receiving messages - 1->3 MsgAppResp Term:2 Log:0/12 - 2->3 MsgAppResp Term:2 Log:0/12 + 1->3 MsgAppResp Term:2 Log:0/12 Commit:12 + 2->3 MsgAppResp Term:2 Log:0/12 Commit:12 # Node 3 is now the leader. Even though the leader is active, nodes 1 and 2 can # still win a prevote and election if they both explicitly campaign, since the @@ -302,7 +302,7 @@ stabilize Entries: 3/13 EntryNormal "" Messages: - 1->2 MsgAppResp Term:3 Log:0/13 + 1->2 MsgAppResp Term:3 Log:0/13 Commit:12 > 3 handling Ready Ready MustSync=true: Lead:2 State:StateFollower @@ -310,10 +310,10 @@ stabilize Entries: 3/13 EntryNormal "" Messages: - 3->2 MsgAppResp Term:3 Log:0/13 + 3->2 MsgAppResp Term:3 Log:0/13 Commit:12 > 2 receiving messages - 1->2 MsgAppResp Term:3 Log:0/13 - 3->2 MsgAppResp Term:3 Log:0/13 + 1->2 MsgAppResp Term:3 Log:0/13 Commit:12 + 3->2 MsgAppResp Term:3 Log:0/13 Commit:12 > 2 handling Ready Ready MustSync=false: HardState Term:3 Vote:2 Commit:13 @@ -332,14 +332,14 @@ stabilize CommittedEntries: 3/13 EntryNormal "" Messages: - 1->2 MsgAppResp Term:3 Log:0/13 + 1->2 MsgAppResp Term:3 Log:0/13 Commit:13 > 3 handling Ready Ready MustSync=false: HardState Term:3 Commit:13 CommittedEntries: 3/13 EntryNormal "" Messages: - 3->2 MsgAppResp Term:3 Log:0/13 + 3->2 MsgAppResp Term:3 Log:0/13 Commit:13 > 2 receiving messages - 1->2 MsgAppResp Term:3 Log:0/13 - 3->2 MsgAppResp Term:3 Log:0/13 + 1->2 MsgAppResp Term:3 Log:0/13 Commit:13 + 3->2 MsgAppResp Term:3 Log:0/13 Commit:13 diff --git a/testdata/probe_and_replicate.txt b/testdata/probe_and_replicate.txt index d970183c..05f17a13 100644 --- a/testdata/probe_and_replicate.txt +++ b/testdata/probe_and_replicate.txt @@ -510,9 +510,9 @@ stabilize 1 2 6/20 EntryNormal "prop_6_20" 8/21 EntryNormal "" Messages: - 2->1 MsgAppResp Term:8 Log:0/21 + 2->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 receiving messages - 2->1 MsgAppResp Term:8 Log:0/21 + 2->1 MsgAppResp Term:8 Log:0/21 Commit:18 stabilize 1 3 ---- @@ -564,9 +564,9 @@ stabilize 1 3 5/17 EntryNormal "prop_5_17" 6/18 EntryNormal "" Messages: - 3->1 MsgAppResp Term:8 Log:0/21 + 3->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 receiving messages - 3->1 MsgAppResp Term:8 Log:0/21 + 3->1 MsgAppResp Term:8 Log:0/21 Commit:18 stabilize 1 4 ---- @@ -580,9 +580,9 @@ stabilize 1 4 Entries: 8/21 EntryNormal "" Messages: - 4->1 MsgAppResp Term:8 Log:0/21 + 4->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 receiving messages - 4->1 MsgAppResp Term:8 Log:0/21 + 4->1 MsgAppResp Term:8 Log:0/21 Commit:18 > 1 handling Ready Ready MustSync=false: HardState Term:8 Vote:1 Commit:21 @@ -604,9 +604,9 @@ stabilize 1 4 6/20 EntryNormal "prop_6_20" 8/21 EntryNormal "" Messages: - 4->1 MsgAppResp Term:8 Log:0/21 + 4->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 4->1 MsgAppResp Term:8 Log:0/21 + 4->1 MsgAppResp Term:8 Log:0/21 Commit:21 stabilize 1 5 ---- @@ -647,9 +647,9 @@ stabilize 1 5 6/20 EntryNormal "prop_6_20" 8/21 EntryNormal "" Messages: - 5->1 MsgAppResp Term:8 Log:0/21 + 5->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 5->1 MsgAppResp Term:8 Log:0/21 + 5->1 MsgAppResp Term:8 Log:0/21 Commit:21 stabilize 1 6 ---- @@ -702,9 +702,9 @@ stabilize 1 6 6/20 EntryNormal "prop_6_20" 8/21 EntryNormal "" Messages: - 6->1 MsgAppResp Term:8 Log:0/21 + 6->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 6->1 MsgAppResp Term:8 Log:0/21 + 6->1 MsgAppResp Term:8 Log:0/21 Commit:21 stabilize 1 7 ---- @@ -765,6 +765,6 @@ stabilize 1 7 6/20 EntryNormal "prop_6_20" 8/21 EntryNormal "" Messages: - 7->1 MsgAppResp Term:8 Log:0/21 + 7->1 MsgAppResp Term:8 Log:0/21 Commit:21 > 1 receiving messages - 7->1 MsgAppResp Term:8 Log:0/21 + 7->1 MsgAppResp Term:8 Log:0/21 Commit:21 diff --git a/tracker/progress.go b/tracker/progress.go index 1b594cf3..905dfedc 100644 --- a/tracker/progress.go +++ b/tracker/progress.go @@ -30,6 +30,8 @@ import ( type Progress struct { // Match is the index up to which the follower's log is known to match the // leader's. + // + // Match does not regress. Match uint64 // Next is the log index of the next entry to send to this follower. All // entries with indices in (Match, Next) interval are already in flight. @@ -40,8 +42,16 @@ type Progress struct { // In StateSnapshot, Next == PendingSnapshot + 1. Next uint64 + // Commit is the known commit index of the follower's log. + // + // Invariant: Commit <= Match. + // Invariant: Commit <= leader's commit index. + // Commit does not regress. + Commit uint64 // sentCommit is the highest commit index in flight to the follower. // + // Invariant: sentCommit >= Commit. + // // In StateSnapshot, sentCommit == PendingSnapshot == Next-1. sentCommit uint64 @@ -196,6 +206,11 @@ func (pr *Progress) SentCommit(commit uint64) { pr.sentCommit = commit } +// UpdateCommit moves the known commit index for this follower forward. +func (pr *Progress) UpdateCommit(index uint64) { + pr.Commit = max(pr.Commit, min(index, pr.Match)) +} + // MaybeUpdate is called when an MsgAppResp arrives from the follower, with the // index acked by it. The method returns false if the given n index comes from // an outdated message. Otherwise it updates the progress and returns true.