add method (*EtcdServer) IsRaftLoopBlocked to support checking whethe…

…r the raft loop is blocked

Signed-off-by: Benjamin Wang <[email protected]>

server/etcdserver/raft.go

@@ -99,6 +99,9 @@ type raftNode struct {
 
 	stopped chan struct{}
 	done    chan struct{}
+
+	// used by liveness probe to check whether the raftloop is blocked.
+	dummyc chan struct{}
 }
 
 type raftNodeConfig struct {
@@ -142,6 +145,7 @@ func newRaftNode(cfg raftNodeConfig) *raftNode {
 		applyc:     make(chan toApply),
 		stopped:    make(chan struct{}),
 		done:       make(chan struct{}),
+		dummyc:     make(chan struct{}),
 	}
 	if r.heartbeat == 0 {
 		r.ticker = &time.Ticker{}
@@ -322,6 +326,8 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 					// notify etcdserver that raft has already been notified or advanced.
 					raftAdvancedC <- struct{}{}
 				}
+			case <-r.dummyc:
+				r.lg.Debug("Received dummy event")
 			case <-r.stopped:
 				return
 			}
@@ -413,6 +419,17 @@ func (r *raftNode) onStop() {
 	close(r.done)
 }
 
+func (r *raftNode) trySendDummyEvent(timeout time.Duration) error {
+	select {
+	case r.dummyc <- struct{}{}:
+	case <-r.done:
+	case <-time.After(timeout):
+		return fmt.Errorf("failed to send dummy event in %s", timeout.String())
+	}
+
+	return nil
+}
+
 // for testing
 func (r *raftNode) pauseSending() {
 	p := r.transport.(rafthttp.Pausable)

server/etcdserver/server.go

@@ -1262,6 +1262,17 @@ func (s *EtcdServer) Stop() {
 	s.HardStop()
 }
 
+// IsRaftLoopBlocked checks whether the raft loop has blocked for at least
+// the duration specified by `timeout`, and it defaults to 2*ElectionTimeout,
+// which is the maximum time to trigger a new leader election.
+// If the returned error isn't nil, then it's blocked; otherwise not.
+func (s *EtcdServer) IsRaftLoopBlocked(timeout time.Duration) error {
+	if timeout == 0 {
+		timeout = 2 * s.Cfg.ElectionTimeout()
+	}
+	return s.r.trySendDummyEvent(timeout)
+}
+
 // ReadyNotify returns a channel that will be closed when the server
 // is ready to serve client requests
 func (s *EtcdServer) ReadyNotify() <-chan struct{} { return s.readych }