raft: send commit index eagerly and only if necessary

Signed-off-by: Pavel Kalinnikov <[email protected]>
pav-kv · Jan 24, 2024 · 303bd94 · 303bd94
1 parent c9ea2f0
commit 303bd94
Show file tree

Hide file tree

Showing 8 changed files with 84 additions and 74 deletions.
diff --git a/raft.go b/raft.go
@@ -1466,7 +1466,6 @@ func stepLeader(r *raft, m pb.Message) error {
 			}
 		} else {
 			pr.UpdateCommit(m.Commit)
-			oldPaused := pr.IsPaused()
 			// We want to update our tracking if the response updates our
 			// matched index or if the response can move a probing peer back
 			// into StateReplicate (see heartbeat_rep_recovers_from_probing.txt
@@ -1500,11 +1499,8 @@ func stepLeader(r *raft, m pb.Message) error {
 					// to respond to pending read index requests
 					releasePendingReadIndexMessages(r)
 					r.bcastAppend()
-				} else if oldPaused {
-					// If we were paused before, this node may be missing the
-					// latest commit index, so send it.
-					// TODO(pav-kv): remove this branch, and decide on sending the commit
-					// index update based on pr.Commit.
+				} else if r.id != m.From && pr.Commit < r.raftLog.committed {
+					// This node is missing the latest commit index, send it.
 					r.sendAppend(m.From)
 				}
 				// We've updated flow control information above, which may

diff --git a/testdata/async_storage_writes.txt b/testdata/async_storage_writes.txt
@@ -327,6 +327,7 @@ process-ready 1 2 3
   Messages:
   1->2 MsgApp Term:1 Log:1/13 Commit:12
   1->3 MsgApp Term:1 Log:1/13 Commit:12
+  1->3 MsgApp Term:1 Log:1/13 Commit:12
   1->2 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"]
   1->3 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"]
   1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[1->1 MsgAppResp Term:1 Log:0/14 Commit:12, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14]
@@ -341,6 +342,7 @@ deliver-msgs 1 2 3
 1->2 MsgApp Term:1 Log:1/13 Commit:12
 1->2 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"]
 1->3 MsgApp Term:1 Log:1/13 Commit:12
+1->3 MsgApp Term:1 Log:1/13 Commit:12
 1->3 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"]
 
 process-ready 1 2 3
@@ -365,7 +367,7 @@ process-ready 1 2 3
   CommittedEntries:
   1/12 EntryNormal "prop_1"
   Messages:
-  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[3->1 MsgAppResp Term:1 Log:0/13 Commit:12, 3->1 MsgAppResp Term:1 Log:0/14 Commit:12, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14]
+  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[3->1 MsgAppResp Term:1 Log:0/13 Commit:12, 3->1 MsgAppResp Term:1 Log:0/13 Commit:12, 3->1 MsgAppResp Term:1 Log:0/14 Commit:12, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14]
   3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"]]
 
 process-append-thread 1 2 3
@@ -414,6 +416,7 @@ process-ready 1 2 3
   Messages:
   1->2 MsgApp Term:1 Log:1/14 Commit:13
   1->3 MsgApp Term:1 Log:1/14 Commit:13
+  1->3 MsgApp Term:1 Log:1/14 Commit:13
   1->2 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"]
   1->3 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"]
   1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[1->1 MsgAppResp Term:1 Log:0/15 Commit:13, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15]
@@ -428,6 +431,7 @@ deliver-msgs 1 2 3
 1->2 MsgApp Term:1 Log:1/14 Commit:13
 1->2 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"]
 1->3 MsgApp Term:1 Log:1/14 Commit:13
+1->3 MsgApp Term:1 Log:1/14 Commit:13
 1->3 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"]
 
 process-ready 1 2 3
@@ -452,7 +456,7 @@ process-ready 1 2 3
   CommittedEntries:
   1/13 EntryNormal "prop_2"
   Messages:
-  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[3->1 MsgAppResp Term:1 Log:0/14 Commit:13, 3->1 MsgAppResp Term:1 Log:0/15 Commit:13, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15]
+  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[3->1 MsgAppResp Term:1 Log:0/14 Commit:13, 3->1 MsgAppResp Term:1 Log:0/14 Commit:13, 3->1 MsgAppResp Term:1 Log:0/15 Commit:13, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15]
   3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"]]
 
 process-append-thread 1 2 3
@@ -475,6 +479,7 @@ process-append-thread 1 2 3
   3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"]
   Responses:
   3->1 MsgAppResp Term:1 Log:0/13 Commit:12
+  3->1 MsgAppResp Term:1 Log:0/13 Commit:12
   3->1 MsgAppResp Term:1 Log:0/14 Commit:12
   AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14
 
@@ -503,6 +508,7 @@ AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14
 2->1 MsgAppResp Term:1 Log:0/13 Commit:12
 2->1 MsgAppResp Term:1 Log:0/14 Commit:12
 3->1 MsgAppResp Term:1 Log:0/13 Commit:12
+3->1 MsgAppResp Term:1 Log:0/13 Commit:12
 3->1 MsgAppResp Term:1 Log:0/14 Commit:12
 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"]
 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14
@@ -520,6 +526,7 @@ process-ready 1 2 3
   Messages:
   1->2 MsgApp Term:1 Log:1/15 Commit:14
   1->3 MsgApp Term:1 Log:1/15 Commit:14
+  1->3 MsgApp Term:1 Log:1/15 Commit:14
   1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15]
   1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]]
 > 2 handling Ready
@@ -531,6 +538,7 @@ deliver-msgs 1 2 3
 ----
 1->2 MsgApp Term:1 Log:1/15 Commit:14
 1->3 MsgApp Term:1 Log:1/15 Commit:14
+1->3 MsgApp Term:1 Log:1/15 Commit:14
 
 process-ready 1 2 3
 ----
@@ -550,7 +558,7 @@ process-ready 1 2 3
   CommittedEntries:
   1/14 EntryNormal "prop_3"
   Messages:
-  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15 Commit:14, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15]
+  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15 Commit:14, 3->1 MsgAppResp Term:1 Log:0/15 Commit:14, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15]
   3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]]
 
 process-append-thread 1 2 3
@@ -573,6 +581,7 @@ process-append-thread 1 2 3
   3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"]
   Responses:
   3->1 MsgAppResp Term:1 Log:0/14 Commit:13
+  3->1 MsgAppResp Term:1 Log:0/14 Commit:13
   3->1 MsgAppResp Term:1 Log:0/15 Commit:13
   AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15
 
@@ -601,6 +610,7 @@ AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15
 2->1 MsgAppResp Term:1 Log:0/14 Commit:13
 2->1 MsgAppResp Term:1 Log:0/15 Commit:13
 3->1 MsgAppResp Term:1 Log:0/14 Commit:13
+3->1 MsgAppResp Term:1 Log:0/14 Commit:13
 3->1 MsgAppResp Term:1 Log:0/15 Commit:13
 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"]
 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15
@@ -618,6 +628,7 @@ process-ready 1 2 3
   Messages:
   1->2 MsgApp Term:1 Log:1/15 Commit:15
   1->3 MsgApp Term:1 Log:1/15 Commit:15
+  1->3 MsgApp Term:1 Log:1/15 Commit:15
   1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1
   1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]]
 > 2 handling Ready
@@ -629,6 +640,7 @@ deliver-msgs 1 2 3
 ----
 1->2 MsgApp Term:1 Log:1/15 Commit:15
 1->3 MsgApp Term:1 Log:1/15 Commit:15
+1->3 MsgApp Term:1 Log:1/15 Commit:15
 
 process-ready 1 2 3
 ----
@@ -648,7 +660,7 @@ process-ready 1 2 3
   CommittedEntries:
   1/15 EntryNormal "prop_4"
   Messages:
-  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15 Commit:15]
+  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15 Commit:15, 3->1 MsgAppResp Term:1 Log:0/15 Commit:15]
   3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]]
 
 process-append-thread 2 3
@@ -664,6 +676,7 @@ process-append-thread 2 3
   3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1
   Responses:
   3->1 MsgAppResp Term:1 Log:0/15 Commit:14
+  3->1 MsgAppResp Term:1 Log:0/15 Commit:14
   AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15
 
 process-apply-thread 1 2 3
@@ -688,6 +701,7 @@ deliver-msgs 1 2 3
 ----
 2->1 MsgAppResp Term:1 Log:0/15 Commit:14
 3->1 MsgAppResp Term:1 Log:0/15 Commit:14
+3->1 MsgAppResp Term:1 Log:0/15 Commit:14
 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]
 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15
 INFO entry at index 15 missing from unstable log; ignoring
@@ -717,6 +731,7 @@ process-append-thread 2 3
   3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1
   Responses:
   3->1 MsgAppResp Term:1 Log:0/15 Commit:15
+  3->1 MsgAppResp Term:1 Log:0/15 Commit:15
 
 process-apply-thread 1 2 3
 ----
@@ -740,6 +755,7 @@ deliver-msgs 1 2 3
 ----
 2->1 MsgAppResp Term:1 Log:0/15 Commit:15
 3->1 MsgAppResp Term:1 Log:0/15 Commit:15
+3->1 MsgAppResp Term:1 Log:0/15 Commit:15
 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]
 ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]
 ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]

diff --git a/testdata/confchange_v1_remove_leader.txt b/testdata/confchange_v1_remove_leader.txt
@@ -141,16 +141,26 @@ stabilize 1
   2->1 MsgAppResp Term:1 Log:0/6 Commit:3
   2->1 MsgAppResp Term:1 Log:0/6 Commit:4
   2->1 MsgAppResp Term:1 Log:0/6 Commit:5
+> 1 handling Ready
+  Ready MustSync=false:
+  Messages:
+  1->2 MsgApp Term:1 Log:1/6 Commit:5
 
 # When n3 responds, quorum is reached and everything falls into place.
 stabilize
 ----
+> 2 receiving messages
+  1->2 MsgApp Term:1 Log:1/6 Commit:5
 > 3 receiving messages
   1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1]
   1->3 MsgApp Term:1 Log:1/4 Commit:3 Entries:[1/5 EntryNormal "foo"]
   1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"]
   1->3 MsgApp Term:1 Log:1/6 Commit:4
   1->3 MsgApp Term:1 Log:1/6 Commit:5
+> 2 handling Ready
+  Ready MustSync=false:
+  Messages:
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:5
 > 3 handling Ready
   Ready MustSync=true:
   HardState Term:1 Vote:1 Commit:5
@@ -169,6 +179,7 @@ stabilize
   3->1 MsgAppResp Term:1 Log:0/6 Commit:5
   INFO 3 switched to configuration voters=(2 3)
 > 1 receiving messages
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:5
   3->1 MsgAppResp Term:1 Log:0/4 Commit:3
   3->1 MsgAppResp Term:1 Log:0/5 Commit:3
   3->1 MsgAppResp Term:1 Log:0/6 Commit:3
@@ -180,11 +191,15 @@ stabilize
   CommittedEntries:
   1/6 EntryNormal "bar"
   Messages:
+  1->3 MsgApp Term:1 Log:1/6 Commit:5
+  1->3 MsgApp Term:1 Log:1/6 Commit:5
   1->2 MsgApp Term:1 Log:1/6 Commit:6
   1->3 MsgApp Term:1 Log:1/6 Commit:6
 > 2 receiving messages
   1->2 MsgApp Term:1 Log:1/6 Commit:6
 > 3 receiving messages
+  1->3 MsgApp Term:1 Log:1/6 Commit:5
+  1->3 MsgApp Term:1 Log:1/6 Commit:5
   1->3 MsgApp Term:1 Log:1/6 Commit:6
 > 2 handling Ready
   Ready MustSync=false:
@@ -199,9 +214,13 @@ stabilize
   CommittedEntries:
   1/6 EntryNormal "bar"
   Messages:
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:5
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:5
   3->1 MsgAppResp Term:1 Log:0/6 Commit:6
 > 1 receiving messages
   2->1 MsgAppResp Term:1 Log:0/6 Commit:6
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:5
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:5
   3->1 MsgAppResp Term:1 Log:0/6 Commit:6
 
 # However not all is well. n1 is still leader but unconditionally drops all

diff --git a/testdata/confchange_v2_add_double_auto.txt b/testdata/confchange_v2_add_double_auto.txt
@@ -282,6 +282,7 @@ stabilize 1
   Messages:
   1->2 MsgApp Term:1 Log:1/8 Commit:6
   1->3 MsgApp Term:1 Log:1/8 Commit:6
+  1->3 MsgApp Term:1 Log:1/8 Commit:6
   INFO 1 switched to configuration voters=(1)&&(1 2 3) autoleave
   INFO initiating automatic transition out of joint configuration voters=(1)&&(1 2 3) autoleave
 > 1 handling Ready
@@ -304,6 +305,7 @@ stabilize 2 3
   1->3 MsgApp Term:1 Log:1/6 Commit:5 Entries:[1/7 EntryNormal "foo"]
   1->3 MsgApp Term:1 Log:1/7 Commit:5 Entries:[1/8 EntryNormal "bar"]
   1->3 MsgApp Term:1 Log:1/8 Commit:6
+  1->3 MsgApp Term:1 Log:1/8 Commit:6
   1->3 MsgApp Term:1 Log:1/8 Commit:6 Entries:[1/9 EntryConfChangeV2]
 > 2 handling Ready
   Ready MustSync=true:
@@ -333,6 +335,7 @@ stabilize 2 3
   3->1 MsgAppResp Term:1 Log:0/7 Commit:5
   3->1 MsgAppResp Term:1 Log:0/8 Commit:5
   3->1 MsgAppResp Term:1 Log:0/8 Commit:6
+  3->1 MsgAppResp Term:1 Log:0/8 Commit:6
   3->1 MsgAppResp Term:1 Log:0/9 Commit:6
   INFO 3 switched to configuration voters=(1)&&(1 2 3) autoleave
 
@@ -349,6 +352,7 @@ stabilize
   3->1 MsgAppResp Term:1 Log:0/7 Commit:5
   3->1 MsgAppResp Term:1 Log:0/8 Commit:5
   3->1 MsgAppResp Term:1 Log:0/8 Commit:6
+  3->1 MsgAppResp Term:1 Log:0/8 Commit:6
   3->1 MsgAppResp Term:1 Log:0/9 Commit:6
 > 1 handling Ready
   Ready MustSync=false:
@@ -364,6 +368,9 @@ stabilize
   1->3 MsgApp Term:1 Log:1/9 Commit:8
   1->2 MsgApp Term:1 Log:1/9 Commit:9
   1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
   INFO 1 switched to configuration voters=(1)
 > 2 receiving messages
   1->2 MsgApp Term:1 Log:1/9 Commit:7
@@ -373,6 +380,9 @@ stabilize
   1->3 MsgApp Term:1 Log:1/9 Commit:7
   1->3 MsgApp Term:1 Log:1/9 Commit:8
   1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
 > 2 handling Ready
   Ready MustSync=false:
   HardState Term:1 Commit:9
@@ -396,6 +406,9 @@ stabilize
   3->1 MsgAppResp Term:1 Log:0/9 Commit:7
   3->1 MsgAppResp Term:1 Log:0/9 Commit:8
   3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
   INFO 3 switched to configuration voters=(1)
 > 1 receiving messages
   2->1 MsgAppResp Term:1 Log:0/9 Commit:7
@@ -410,3 +423,9 @@ stabilize
   raft: cannot step as peer not found
   3->1 MsgAppResp Term:1 Log:0/9 Commit:9
   raft: cannot step as peer not found
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  raft: cannot step as peer not found
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  raft: cannot step as peer not found
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  raft: cannot step as peer not found