diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index 8242a4f97a52..0b35df165fa8 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -888,6 +888,9 @@ func (s *EtcdServer) revokeExpiredLeases(leases []*lease.Lease) { // ensureLeadership checks whether current member is still the leader. func (s *EtcdServer) ensureLeadership() bool { + // gofail: var ensureLeadershipRetVal bool + // return ensureLeadershipRetVal + lg := s.Logger() ctx, cancel := context.WithTimeout(s.ctx, s.Cfg.ReqTimeout()) diff --git a/tests/e2e/ctl_v3_lease_test.go b/tests/e2e/ctl_v3_lease_test.go index 3c8887627150..9f5554a5f806 100644 --- a/tests/e2e/ctl_v3_lease_test.go +++ b/tests/e2e/ctl_v3_lease_test.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !cluster_proxy + package e2e import ( @@ -114,7 +116,7 @@ func TestLeaseRevokeByOldLeader(t *testing.T) { }) } -// TestLeaseRevokeByOldLeader verifies that leases shouldn't be revoked by +// TestLeaseRevokeByNewLeader verifies that leases shouldn't be revoked by // new leader. // See the second case in the section "Root Cause" in // https://docs.google.com/document/d/1peLDjwebnSuNR69ombuUiJ5fkSw7Rd3Slv_mUpOLsmg/edit @@ -234,3 +236,53 @@ func testLeaseRevokeIssue(t *testing.T, epsFn func(cluster *e2e.EtcdProcessClust close(stopC) <-doneC } + +// TestLeaseRetryRevoke verifies that etcd will retry revoking lease +// if failed due to temporary network issue. +func TestLeaseRetryRevoke(t *testing.T) { + e2e.BeforeTest(t) + + t.Log("Starting a new etcd cluster") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + epc, err := e2e.NewEtcdProcessCluster(ctx, t, + e2e.WithClusterSize(1), + e2e.WithEnvVars(map[string]string{"GOFAIL_FAILPOINTS": `ensureLeadershipRetVal=1*return(false)`}), + ) + cancel() + require.NoError(t, err) + defer func() { + if errC := epc.Close(); errC != nil { + t.Fatalf("error closing etcd processes (%v)", errC) + } + }() + + t.Log("Creating a client") + client, err := clientv3.New(clientv3.Config{Endpoints: epc.EndpointsGRPC(), DialTimeout: 3 * time.Second}) + require.NoError(t, err) + defer client.Close() + + t.Log("Creating a new lease") + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + _, err = client.Grant(ctx, 5) + cancel() + require.NoError(t, err) + + t.Log("Checking lease revoking request being ignored") + ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, err = epc.Procs[0].Logs().ExpectWithContext(ctx, expect.ExpectedResponse{Value: "Ignore the lease revoking request"}) + require.NoError(t, err, "can't get log indicating lease revoking request being ignored") + + t.Log("Confirm the lease will eventually be revoked") + testutils.ExecuteUntil(ctx, t, func() { + for { + leases, lerr := client.Leases(ctx) + require.NoError(t, lerr) + if len(leases.Leases) == 0 { + t.Log("The lease has been revoked") + break + } + time.Sleep(100 * time.Millisecond) + } + }) +}