raft,tracker: track commit index of each follower

Signed-off-by: Pavel Kalinnikov <[email protected]>
etcd-io · Jan 24, 2024 · da6ad00 · da6ad00
1 parent 11c051a
commit da6ad00
Show file tree

Hide file tree

Showing 22 changed files with 331 additions and 317 deletions.
diff --git a/raft.go b/raft.go
@@ -815,7 +815,7 @@ func (r *raft) appendEntry(es ...pb.Entry) (accepted bool) {
 	//  if r.maybeCommit() {
 	//  	r.bcastAppend()
 	//  }
-	r.send(pb.Message{To: r.id, Type: pb.MsgAppResp, Index: li})
+	r.send(pb.Message{To: r.id, Type: pb.MsgAppResp, Index: li, Commit: r.raftLog.committed})
 	return true
 }
 
@@ -1465,6 +1465,7 @@ func stepLeader(r *raft, m pb.Message) error {
 				r.sendAppend(m.From)
 			}
 		} else {
+			pr.UpdateCommit(m.Commit)
 			oldPaused := pr.IsPaused()
 			// We want to update our tracking if the response updates our
 			// matched index or if the response can move a probing peer back
@@ -1505,6 +1506,8 @@ func stepLeader(r *raft, m pb.Message) error {
 				} else if oldPaused {
 					// If we were paused before, this node may be missing the
 					// latest commit index, so send it.
+					// TODO(pav-kv): remove this branch, and decide on sending the commit
+					// index update based on pr.Commit.
 					r.sendAppend(m.From)
 				}
 				// We've updated flow control information above, which may
@@ -1730,12 +1733,12 @@ func stepFollower(r *raft, m pb.Message) error {
 }
 
 func (r *raft) handleAppendEntries(m pb.Message) {
-	if m.Index < r.raftLog.committed {
-		r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: r.raftLog.committed})
+	if committed := r.raftLog.committed; m.Index < committed {
+		r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: committed, Commit: committed})
 		return
 	}
 	if mlastIndex, ok := r.raftLog.maybeAppend(m.Index, m.LogTerm, m.Commit, m.Entries...); ok {
-		r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: mlastIndex})
+		r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: mlastIndex, Commit: m.Commit})
 		return
 	}
 	r.logger.Debugf("%x [logterm: %d, index: %d] rejected MsgApp [logterm: %d, index: %d] from %x",

diff --git a/testdata/async_storage_writes.txt b/testdata/async_storage_writes.txt
diff --git a/testdata/async_storage_writes_append_aba_race.txt b/testdata/async_storage_writes_append_aba_race.txt
@@ -42,7 +42,7 @@ Messages:
 2->5 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"]
 2->6 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"]
 2->7 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"]
-2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[2->2 MsgAppResp Term:1 Log:0/12, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12]
+2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[2->2 MsgAppResp Term:1 Log:0/12 Commit:11, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12]
 
 deliver-msgs 1 drop=(3,4,5,6,7)
 ----
@@ -61,7 +61,7 @@ Ready MustSync=true:
 Entries:
 1/12 EntryNormal "init_prop"
 Messages:
-1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[1->2 MsgAppResp Term:1 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12]
+1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[1->2 MsgAppResp Term:1 Log:0/12 Commit:11, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12]
 
 # Step 4: node 3 becomes the leader after getting a vote from nodes 4, 5, and 6.
 
@@ -183,7 +183,7 @@ Messages:
 3->5 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""]
 3->6 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""]
 3->7 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""]
-3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[2/12 EntryNormal ""] Responses:[3->3 MsgAppResp Term:2 Log:0/12, AppendThread->3 MsgStorageAppendResp Term:2 Log:2/12]
+3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[2/12 EntryNormal ""] Responses:[3->3 MsgAppResp Term:2 Log:0/12 Commit:11, AppendThread->3 MsgStorageAppendResp Term:2 Log:2/12]
 
 deliver-msgs 1 drop=(2,4,5,6,7)
 ----
@@ -210,7 +210,7 @@ HardState Term:2 Commit:11
 Entries:
 2/12 EntryNormal ""
 Messages:
-1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] Responses:[1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0), 1->3 MsgAppResp Term:2 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12]
+1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] Responses:[1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0), 1->3 MsgAppResp Term:2 Log:0/12 Commit:11, AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12]
 
 # Step 6: node 3 crashes and node 4 becomes leader getting the vote from 5, 6, and 7.
 
@@ -325,7 +325,7 @@ Messages:
 4->5 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""]
 4->6 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""]
 4->7 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""]
-4->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[4->4 MsgAppResp Term:3 Log:0/12, AppendThread->4 MsgStorageAppendResp Term:3 Log:3/12]
+4->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[4->4 MsgAppResp Term:3 Log:0/12 Commit:11, AppendThread->4 MsgStorageAppendResp Term:3 Log:3/12]
 
 # Step 7: before the new entries reach node 1, it hears of the term change
 # through a heartbeat and persists the new term. Node 1 then receives these
@@ -390,7 +390,7 @@ Ready MustSync=true:
 Entries:
 3/12 EntryNormal ""
 Messages:
-1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[1->4 MsgAppResp Term:3 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12]
+1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[1->4 MsgAppResp Term:3 Log:0/12 Commit:11, AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12]
 
 # Step 8: The asynchronous log appends from the first Ready complete and the
 # MsgStorageAppendResp is returned to the raft node state machine. A decision
@@ -405,7 +405,7 @@ process-append-thread 1
 Processing:
 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"]
 Responses:
-1->2 MsgAppResp Term:1 Log:0/12
+1->2 MsgAppResp Term:1 Log:0/12 Commit:11
 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12
 
 raft-log 1
@@ -432,7 +432,7 @@ Processing:
 1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""]
 Responses:
 1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0)
-1->3 MsgAppResp Term:2 Log:0/12
+1->3 MsgAppResp Term:2 Log:0/12 Commit:11
 AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12
 
 raft-log 1
@@ -467,7 +467,7 @@ process-append-thread 1
 Processing:
 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""]
 Responses:
-1->4 MsgAppResp Term:3 Log:0/12
+1->4 MsgAppResp Term:3 Log:0/12 Commit:11
 AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12
 
 raft-log 1

diff --git a/testdata/campaign.txt b/testdata/campaign.txt
@@ -76,17 +76,17 @@ stabilize
   Entries:
   1/3 EntryNormal ""
   Messages:
-  2->1 MsgAppResp Term:1 Log:0/3
+  2->1 MsgAppResp Term:1 Log:0/3 Commit:2
 > 3 handling Ready
   Ready MustSync=true:
   Lead:1 State:StateFollower
   Entries:
   1/3 EntryNormal ""
   Messages:
-  3->1 MsgAppResp Term:1 Log:0/3
+  3->1 MsgAppResp Term:1 Log:0/3 Commit:2
 > 1 receiving messages
-  2->1 MsgAppResp Term:1 Log:0/3
-  3->1 MsgAppResp Term:1 Log:0/3
+  2->1 MsgAppResp Term:1 Log:0/3 Commit:2
+  3->1 MsgAppResp Term:1 Log:0/3 Commit:2
 > 1 handling Ready
   Ready MustSync=false:
   HardState Term:1 Vote:1 Commit:3
@@ -105,14 +105,14 @@ stabilize
   CommittedEntries:
   1/3 EntryNormal ""
   Messages:
-  2->1 MsgAppResp Term:1 Log:0/3
+  2->1 MsgAppResp Term:1 Log:0/3 Commit:3
 > 3 handling Ready
   Ready MustSync=false:
   HardState Term:1 Vote:1 Commit:3
   CommittedEntries:
   1/3 EntryNormal ""
   Messages:
-  3->1 MsgAppResp Term:1 Log:0/3
+  3->1 MsgAppResp Term:1 Log:0/3 Commit:3
 > 1 receiving messages
-  2->1 MsgAppResp Term:1 Log:0/3
-  3->1 MsgAppResp Term:1 Log:0/3
+  2->1 MsgAppResp Term:1 Log:0/3 Commit:3
+  3->1 MsgAppResp Term:1 Log:0/3 Commit:3
diff --git a/testdata/campaign_learner_must_vote.txt b/testdata/campaign_learner_must_vote.txt
@@ -129,10 +129,10 @@ stabilize 2 3
   CommittedEntries:
   1/4 EntryConfChangeV2 v3
   Messages:
-  3->2 MsgAppResp Term:2 Log:0/5
+  3->2 MsgAppResp Term:2 Log:0/5 Commit:4
   INFO 3 switched to configuration voters=(1 2 3)
 > 2 receiving messages
-  3->2 MsgAppResp Term:2 Log:0/5
+  3->2 MsgAppResp Term:2 Log:0/5 Commit:4
 > 2 handling Ready
   Ready MustSync=false:
   HardState Term:2 Vote:2 Commit:5
@@ -148,6 +148,6 @@ stabilize 2 3
   CommittedEntries:
   2/5 EntryNormal ""
   Messages:
-  3->2 MsgAppResp Term:2 Log:0/5
+  3->2 MsgAppResp Term:2 Log:0/5 Commit:5
 > 2 receiving messages
-  3->2 MsgAppResp Term:2 Log:0/5
+  3->2 MsgAppResp Term:2 Log:0/5 Commit:5
diff --git a/testdata/checkquorum.txt b/testdata/checkquorum.txt
@@ -193,18 +193,18 @@ stabilize
   Entries:
   3/12 EntryNormal ""
   Messages:
-  1->2 MsgAppResp Term:3 Log:0/12
+  1->2 MsgAppResp Term:3 Log:0/12 Commit:11
 > 3 handling Ready
   Ready MustSync=true:
   Lead:2 State:StateFollower
   HardState Term:3 Commit:11
   Entries:
   3/12 EntryNormal ""
   Messages:
-  3->2 MsgAppResp Term:3 Log:0/12
+  3->2 MsgAppResp Term:3 Log:0/12 Commit:11
 > 2 receiving messages
-  1->2 MsgAppResp Term:3 Log:0/12
-  3->2 MsgAppResp Term:3 Log:0/12
+  1->2 MsgAppResp Term:3 Log:0/12 Commit:11
+  3->2 MsgAppResp Term:3 Log:0/12 Commit:11
 > 2 handling Ready
   Ready MustSync=false:
   HardState Term:3 Vote:2 Commit:12
@@ -223,14 +223,14 @@ stabilize
   CommittedEntries:
   3/12 EntryNormal ""
   Messages:
-  1->2 MsgAppResp Term:3 Log:0/12
+  1->2 MsgAppResp Term:3 Log:0/12 Commit:12
 > 3 handling Ready
   Ready MustSync=false:
   HardState Term:3 Commit:12
   CommittedEntries:
   3/12 EntryNormal ""
   Messages:
-  3->2 MsgAppResp Term:3 Log:0/12
+  3->2 MsgAppResp Term:3 Log:0/12 Commit:12
 > 2 receiving messages
-  1->2 MsgAppResp Term:3 Log:0/12
-  3->2 MsgAppResp Term:3 Log:0/12
+  1->2 MsgAppResp Term:3 Log:0/12 Commit:12
+  3->2 MsgAppResp Term:3 Log:0/12 Commit:12
diff --git a/testdata/confchange_v1_add_single.txt b/testdata/confchange_v1_add_single.txt
@@ -101,6 +101,6 @@ stabilize
 > 2 handling Ready
   Ready MustSync=false:
   Messages:
-  2->1 MsgAppResp Term:1 Log:0/4
+  2->1 MsgAppResp Term:1 Log:0/4 Commit:4
 > 1 receiving messages
-  2->1 MsgAppResp Term:1 Log:0/4
+  2->1 MsgAppResp Term:1 Log:0/4 Commit:4
diff --git a/testdata/confchange_v1_remove_leader.txt b/testdata/confchange_v1_remove_leader.txt
@@ -70,8 +70,8 @@ stabilize 2
   1/4 EntryConfChange r1
   1/5 EntryNormal "foo"
   Messages:
-  2->1 MsgAppResp Term:1 Log:0/4
-  2->1 MsgAppResp Term:1 Log:0/5
+  2->1 MsgAppResp Term:1 Log:0/4 Commit:3
+  2->1 MsgAppResp Term:1 Log:0/5 Commit:3
 
 # Put another entry in n1's log.
 propose 1 bar
@@ -92,8 +92,8 @@ stabilize 1
   1->2 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"]
   1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"]
 > 1 receiving messages
-  2->1 MsgAppResp Term:1 Log:0/4
-  2->1 MsgAppResp Term:1 Log:0/5
+  2->1 MsgAppResp Term:1 Log:0/4 Commit:3
+  2->1 MsgAppResp Term:1 Log:0/5 Commit:3
 > 1 handling Ready
   Ready MustSync=false:
   HardState Term:1 Vote:1 Commit:5
@@ -129,18 +129,18 @@ stabilize 2
   1/4 EntryConfChange r1
   1/5 EntryNormal "foo"
   Messages:
-  2->1 MsgAppResp Term:1 Log:0/6
-  2->1 MsgAppResp Term:1 Log:0/6
-  2->1 MsgAppResp Term:1 Log:0/6
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:3
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:4
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:5
   INFO 2 switched to configuration voters=(2 3)
 
 # ... which thankfully is what we see on the leader.
 stabilize 1
 ----
 > 1 receiving messages
-  2->1 MsgAppResp Term:1 Log:0/6
-  2->1 MsgAppResp Term:1 Log:0/6
-  2->1 MsgAppResp Term:1 Log:0/6
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:3
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:4
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:5
 
 # When n3 responds, quorum is reached and everything falls into place.
 stabilize
@@ -162,18 +162,18 @@ stabilize
   1/4 EntryConfChange r1
   1/5 EntryNormal "foo"
   Messages:
-  3->1 MsgAppResp Term:1 Log:0/4
-  3->1 MsgAppResp Term:1 Log:0/5
-  3->1 MsgAppResp Term:1 Log:0/6
-  3->1 MsgAppResp Term:1 Log:0/6
-  3->1 MsgAppResp Term:1 Log:0/6
+  3->1 MsgAppResp Term:1 Log:0/4 Commit:3
+  3->1 MsgAppResp Term:1 Log:0/5 Commit:3
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:3
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:4
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:5
   INFO 3 switched to configuration voters=(2 3)
 > 1 receiving messages
-  3->1 MsgAppResp Term:1 Log:0/4
-  3->1 MsgAppResp Term:1 Log:0/5
-  3->1 MsgAppResp Term:1 Log:0/6
-  3->1 MsgAppResp Term:1 Log:0/6
-  3->1 MsgAppResp Term:1 Log:0/6
+  3->1 MsgAppResp Term:1 Log:0/4 Commit:3
+  3->1 MsgAppResp Term:1 Log:0/5 Commit:3
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:3
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:4
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:5
 > 1 handling Ready
   Ready MustSync=false:
   HardState Term:1 Vote:1 Commit:6
@@ -192,17 +192,17 @@ stabilize
   CommittedEntries:
   1/6 EntryNormal "bar"
   Messages:
-  2->1 MsgAppResp Term:1 Log:0/6
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:6
 > 3 handling Ready
   Ready MustSync=false:
   HardState Term:1 Vote:1 Commit:6
   CommittedEntries:
   1/6 EntryNormal "bar"
   Messages:
-  3->1 MsgAppResp Term:1 Log:0/6
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:6
 > 1 receiving messages
-  2->1 MsgAppResp Term:1 Log:0/6
-  3->1 MsgAppResp Term:1 Log:0/6
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:6
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:6
 
 # However not all is well. n1 is still leader but unconditionally drops all
 # proposals on the floor, so we're effectively stuck if it still heartbeats