Skip to content

Commit

Permalink
etcdserver: ensure hardstate is persisten before applying committed e…
Browse files Browse the repository at this point in the history
…ntries

Signed-off-by: Benjamin Wang <[email protected]>
  • Loading branch information
ahrtr committed Sep 30, 2023
1 parent 0c0128a commit b53e4f6
Showing 1 changed file with 23 additions and 3 deletions.
26 changes: 23 additions & 3 deletions server/etcdserver/raft.go
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,14 @@ func (r *raftNode) start(rh *raftReadyHandler) {

updateCommittedIndex(&ap, rh)

waitWALSync := shouldWaitWALSync(rd)
if waitWALSync {
// gofail: var raftBeforeSaveWaitWalSync struct{}
if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
r.lg.Fatal("failed to save Raft hard state and entries", zap.Error(err))
}
}

select {
case r.applyc <- ap:
case <-r.stopped:
Expand All @@ -240,9 +248,11 @@ func (r *raftNode) start(rh *raftReadyHandler) {
// gofail: var raftAfterSaveSnap struct{}
}

// gofail: var raftBeforeSave struct{}
if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
r.lg.Fatal("failed to save Raft hard state and entries", zap.Error(err))
if !waitWALSync {
// gofail: var raftBeforeSave struct{}
if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
r.lg.Fatal("failed to save Raft hard state and entries", zap.Error(err))
}
}
if !raft.IsEmptyHardState(rd.HardState) {
proposalsCommitted.Set(float64(rd.HardState.Commit))
Expand Down Expand Up @@ -329,6 +339,16 @@ func (r *raftNode) start(rh *raftReadyHandler) {
}()
}

// If the hardstate isn't empty, we must sync the hardstate to WAL file before
// applying the committed entries. Otherwise, etcd may have already responded
// to the client, but the hardstate isn't persisted to the WAL file. When etcd
// starts again, it loads the stale hardstate, and accordingly a linearizable
// request may get the stale readIndex (actually committed Index), eventually
// it won't wait for the latest applied data before responding to the client.
func shouldWaitWALSync(rd raft.Ready) bool {
return !raft.IsEmptyHardState(rd.HardState) && len(rd.CommittedEntries) > 0
}

func updateCommittedIndex(ap *toApply, rh *raftReadyHandler) {
var ci uint64
if len(ap.entries) != 0 {
Expand Down

0 comments on commit b53e4f6

Please sign in to comment.