diff --git a/tests/e2e/ctl_v3_auth_cluster_test.go b/tests/e2e/ctl_v3_auth_cluster_test.go index 965f81d0f7d..74ef3c4b555 100644 --- a/tests/e2e/ctl_v3_auth_cluster_test.go +++ b/tests/e2e/ctl_v3_auth_cluster_test.go @@ -64,7 +64,7 @@ func TestAuthCluster(t *testing.T) { } // start second process - if err := epc.StartNewProc(ctx, nil, t, rootUserClientOpts); err != nil { + if _, err := epc.StartNewProc(ctx, nil, t, false /* addAsLearner */, rootUserClientOpts); err != nil { t.Fatalf("could not start second etcd process (%v)", err) } diff --git a/tests/e2e/etcd_grpcproxy_test.go b/tests/e2e/etcd_grpcproxy_test.go index db9ad7b4016..fd3353ce6af 100644 --- a/tests/e2e/etcd_grpcproxy_test.go +++ b/tests/e2e/etcd_grpcproxy_test.go @@ -64,7 +64,7 @@ func TestGrpcProxyAutoSync(t *testing.T) { require.NoError(t, err) // Add and start second member - err = epc.StartNewProc(ctx, nil, t) + _, err = epc.StartNewProc(ctx, nil, t, false /* addAsLearner */) require.NoError(t, err) // Wait for auto sync of endpoints diff --git a/tests/e2e/etcd_mix_versions_test.go b/tests/e2e/etcd_mix_versions_test.go index e4b86096383..de3f51cff39 100644 --- a/tests/e2e/etcd_mix_versions_test.go +++ b/tests/e2e/etcd_mix_versions_test.go @@ -92,7 +92,7 @@ func mixVersionsSnapshotTestByAddingMember(t *testing.T, clusterVersion, newInst newCfg.Version = newInstanceVersion newCfg.SnapshotCatchUpEntries = 10 t.Log("Starting a new etcd instance") - err = epc.StartNewProc(context.TODO(), &newCfg, t) + _, err = epc.StartNewProc(context.TODO(), &newCfg, t, false /* addAsLearner */) require.NoError(t, err, "failed to start the new etcd instance: %v", err) defer epc.CloseProc(context.TODO(), nil) diff --git a/tests/e2e/runtime_reconfiguration_test.go b/tests/e2e/runtime_reconfiguration_test.go new file mode 100644 index 00000000000..eaf1e528c91 --- /dev/null +++ b/tests/e2e/runtime_reconfiguration_test.go @@ -0,0 +1,200 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !cluster_proxy + +package e2e + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd/server/v3/etcdserver" + "go.etcd.io/etcd/tests/v3/framework/e2e" +) + +// TestRuntimeReconfigGrowClusterSize ensures growing cluster size with two phases +// Phase 1 - Inform cluster of new configuration +// Phase 2 - Start new member +func TestRuntimeReconfigGrowClusterSize(t *testing.T) { + e2e.BeforeTest(t) + + tcs := []struct { + name string + clusterSize int + asLearner bool + }{ + { + name: "grow cluster size from 1 to 3", + clusterSize: 1, + }, + { + name: "grow cluster size from 3 to 5", + clusterSize: 3, + }, + { + name: "grow cluster size from 1 to 3 with learner", + clusterSize: 1, + asLearner: true, + }, + { + name: "grow cluster size from 3 to 5 with learner", + clusterSize: 3, + asLearner: true, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(tc.clusterSize)) + require.NoError(t, err) + require.NoError(t, epc.Procs[0].Etcdctl().Health(ctx)) + defer func() { + err := epc.Close() + require.NoError(t, err, "failed to close etcd cluster: %v", err) + }() + + for i := 0; i < 2; i++ { + time.Sleep(etcdserver.HealthInterval) + if !tc.asLearner { + addMember(ctx, t, epc) + } else { + addMemberAsLearnerAndPromote(ctx, t, epc) + } + } + }) + } +} + +func TestRuntimeReconfigDecreaseClusterSize(t *testing.T) { + e2e.BeforeTest(t) + + tcs := []struct { + name string + clusterSize int + asLearner bool + }{ + { + name: "decrease cluster size from 3 to 1", + clusterSize: 3, + }, + { + name: "decrease cluster size from 5 to 3", + clusterSize: 5, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(tc.clusterSize)) + require.NoError(t, err) + require.NoError(t, epc.Procs[0].Etcdctl().Health(ctx)) + defer func() { + err := epc.Close() + require.NoError(t, err, "failed to close etcd cluster: %v", err) + }() + + for i := 0; i < 2; i++ { + time.Sleep(etcdserver.HealthInterval) + removeFirstMember(ctx, t, epc) + } + }) + } +} + +func TestRuntimeReconfigRollingUpgrade(t *testing.T) { + e2e.BeforeTest(t) + + tcs := []struct { + name string + withLearner bool + }{ + { + name: "with learner", + withLearner: true, + }, + { + name: "without learner", + withLearner: false, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(3)) + require.NoError(t, err) + require.NoError(t, epc.Procs[0].Etcdctl().Health(ctx)) + defer func() { + err := epc.Close() + require.NoError(t, err, "failed to close etcd cluster: %v", err) + }() + + for i := 0; i < 2; i++ { + time.Sleep(etcdserver.HealthInterval) + removeFirstMember(ctx, t, epc) + epc.WaitLeader(t) + // if we do not wait for leader, without the fix of notify raft Advance, + // have to wait 1 sec to pass the test stably. + if tc.withLearner { + addMemberAsLearnerAndPromote(ctx, t, epc) + } else { + addMember(ctx, t, epc) + } + } + }) + } +} + +func addMember(ctx context.Context, t *testing.T, epc *e2e.EtcdProcessCluster) { + _, err := epc.StartNewProc(ctx, nil, t, false /* addAsLearner */) + require.NoError(t, err) + require.NoError(t, epc.Procs[len(epc.Procs)-1].Etcdctl().Health(ctx)) +} + +func addMemberAsLearnerAndPromote(ctx context.Context, t *testing.T, epc *e2e.EtcdProcessCluster) { + id, err := epc.StartNewProc(ctx, nil, t, true /* addAsLearner */) + require.NoError(t, err) + newLearnerMemberProc := epc.Procs[len(epc.Procs)-1] + _, err = epc.Etcdctl().MemberPromote(ctx, id) + require.NoError(t, err) + require.NoError(t, newLearnerMemberProc.Etcdctl().Health(ctx)) +} + +func removeFirstMember(ctx context.Context, t *testing.T, epc *e2e.EtcdProcessCluster) { + // avoid tearing down the last etcd process + if len(epc.Procs) == 1 { + return + } + + firstProc := epc.Procs[0] + sts, err := firstProc.Etcdctl().Status(ctx) + require.NoError(t, err) + memberIDToRemove := sts[0].Header.MemberId + + epc.Procs = epc.Procs[1:] + _, err = epc.Etcdctl().MemberRemove(ctx, memberIDToRemove) + require.NoError(t, err) + require.NoError(t, firstProc.Stop()) + require.NoError(t, firstProc.Close()) +} diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index 6d2ec701940..f00c66f3544 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -30,6 +30,7 @@ import ( "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/etcdserverpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/proxy" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/config" @@ -780,7 +781,10 @@ func (epc *EtcdProcessCluster) CloseProc(ctx context.Context, finder func(EtcdPr return proc.Close() } -func (epc *EtcdProcessCluster) StartNewProc(ctx context.Context, cfg *EtcdProcessClusterConfig, tb testing.TB, opts ...config.ClientOption) error { +// StartNewProc grows cluster size by one with two phases +// Phase 1 - Inform cluster of new configuration +// Phase 2 - Start new member +func (epc *EtcdProcessCluster) StartNewProc(ctx context.Context, cfg *EtcdProcessClusterConfig, tb testing.TB, addAsLearner bool, opts ...config.ClientOption) (memberID uint64, err error) { var serverCfg *EtcdServerProcessConfig if cfg != nil { serverCfg = cfg.EtcdServerProcessConfig(tb, epc.nextSeq) @@ -800,22 +804,29 @@ func (epc *EtcdProcessCluster) StartNewProc(ctx context.Context, cfg *EtcdProces epc.Cfg.SetInitialOrDiscovery(serverCfg, initialCluster, "existing") // First add new member to cluster + tb.Logf("add new member to cluster; member-name %s, member-peer-url %s", serverCfg.Name, serverCfg.PeerURL.String()) memberCtl := epc.Etcdctl(opts...) - _, err := memberCtl.MemberAdd(ctx, serverCfg.Name, []string{serverCfg.PeerURL.String()}) + var resp *clientv3.MemberAddResponse + if addAsLearner { + resp, err = memberCtl.MemberAddAsLearner(ctx, serverCfg.Name, []string{serverCfg.PeerURL.String()}) + } else { + resp, err = memberCtl.MemberAdd(ctx, serverCfg.Name, []string{serverCfg.PeerURL.String()}) + } if err != nil { - return fmt.Errorf("failed to add new member: %w", err) + return 0, fmt.Errorf("failed to add new member: %w", err) } // Then start process + tb.Log("start new member") proc, err := NewEtcdProcess(serverCfg) if err != nil { epc.Close() - return fmt.Errorf("cannot configure: %v", err) + return 0, fmt.Errorf("cannot configure: %v", err) } epc.Procs = append(epc.Procs, proc) - return proc.Start(ctx) + return resp.Member.ID, proc.Start(ctx) } // UpdateProcOptions updates the options for a specific process. If no opt is set, then the config is identical diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index 031e42c9d35..b716f34a704 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -302,6 +302,12 @@ func (ctl *EtcdctlV3) MemberRemove(ctx context.Context, id uint64) (*clientv3.Me return &resp, err } +func (ctl *EtcdctlV3) MemberPromote(ctx context.Context, id uint64) (*clientv3.MemberPromoteResponse, error) { + var resp clientv3.MemberPromoteResponse + err := ctl.spawnJsonCmd(ctx, &resp, "member", "promote", fmt.Sprintf("%x", id)) + return &resp, err +} + func (ctl *EtcdctlV3) cmdArgs(args ...string) []string { cmdArgs := []string{BinPath.Etcdctl} for k, v := range ctl.flags() {