add test case to verify etcd will retry lease revoke due to temporary…

… network issue

Signed-off-by: Benjamin Wang <[email protected]>

server/etcdserver/server.go

@@ -888,6 +888,9 @@ func (s *EtcdServer) revokeExpiredLeases(leases []*lease.Lease) {
 
 // ensureLeadership checks whether current member is still the leader.
 func (s *EtcdServer) ensureLeadership() bool {
+	// gofail: var ensureLeadershipRetVal bool
+	// return ensureLeadershipRetVal
+
 	lg := s.Logger()
 
 	ctx, cancel := context.WithTimeout(s.ctx, s.Cfg.ReqTimeout())

tests/e2e/ctl_v3_lease_test.go

@@ -114,7 +114,7 @@ func TestLeaseRevokeByOldLeader(t *testing.T) {
 	})
 }
 
-// TestLeaseRevokeByOldLeader verifies that leases shouldn't be revoked by
+// TestLeaseRevokeByNewLeader verifies that leases shouldn't be revoked by
 // new leader.
 // See the second case in the section "Root Cause" in
 // https://docs.google.com/document/d/1peLDjwebnSuNR69ombuUiJ5fkSw7Rd3Slv_mUpOLsmg/edit
@@ -234,3 +234,53 @@ func testLeaseRevokeIssue(t *testing.T, epsFn func(cluster *e2e.EtcdProcessClust
 	close(stopC)
 	<-doneC
 }
+
+// TestLeaseRetryRevoke verifies that etcd will retry revoking lease
+// if failed due to temporary network issue.
+func TestLeaseRetryRevoke(t *testing.T) {
+	e2e.BeforeTest(t)
+
+	t.Log("Starting a new etcd cluster")
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	epc, err := e2e.NewEtcdProcessCluster(ctx, t,
+		e2e.WithClusterSize(1),
+		e2e.WithEnvVars(map[string]string{"GOFAIL_FAILPOINTS": `ensureLeadershipRetVal=1*return(false)`}),
+	)
+	cancel()
+	require.NoError(t, err)
+	defer func() {
+		if errC := epc.Close(); errC != nil {
+			t.Fatalf("error closing etcd processes (%v)", errC)
+		}
+	}()
+
+	t.Log("Creating a client")
+	client, err := clientv3.New(clientv3.Config{Endpoints: epc.EndpointsGRPC(), DialTimeout: 3 * time.Second})
+	require.NoError(t, err)
+	defer client.Close()
+
+	t.Log("Creating a new lease")
+	ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
+	_, err = client.Grant(ctx, 5)
+	cancel()
+	require.NoError(t, err)
+
+	t.Log("Checking lease revoking request being ignored")
+	ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	_, err = epc.Procs[0].Logs().ExpectWithContext(ctx, expect.ExpectedResponse{Value: "Ignore the lease revoking request"})
+	require.NoError(t, err, "can't get log indicating lease revoking request being ignored")
+
+	t.Log("Confirm the lease will eventually be revoked")
+	testutils.ExecuteUntil(ctx, t, func() {
+		for {
+			leases, lerr := client.Leases(ctx)
+			require.NoError(t, lerr)
+			if len(leases.Leases) == 0 {
+				t.Log("The lease has been revoked")
+				break
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+	})
+}