From 5d321349d30f33b456bebff4c848a76deb837f3d Mon Sep 17 00:00:00 2001 From: Matt DeBoer Date: Mon, 3 Apr 2017 10:37:56 -0700 Subject: [PATCH] add max wait time & expected count for masters --- pkg/cmd/main.go | 52 ++++++++++++++++++++++++++------------ pkg/discovery/discovery.go | 34 ++++++++++++++----------- 2 files changed, 55 insertions(+), 31 deletions(-) diff --git a/pkg/cmd/main.go b/pkg/cmd/main.go index a0e317a..4d93bcc 100644 --- a/pkg/cmd/main.go +++ b/pkg/cmd/main.go @@ -89,6 +89,19 @@ func main() { of the masters (e.g., 'k8s-master-*')`, EnvVar: "ETCDCD_MASTER_NAMES", }, + cli.IntFlag{ + Name: "master-count", + Value: 3, + Usage: `The expected number of masters in an intial cluster; ignored when joining an existing cluster`, + EnvVar: "ETCDCD_MASTER_COUNT", + }, + cli.StringFlag{ + Name: "maximum-master-resolution-time", + Value: "5m", + Usage: `The minimum amount of seconds after which it is viable to join an existing cluster in which + the current node is an expected member`, + EnvVar: "ETCDCD_MAX_MASTER_RESOLUTION_TIME", + }, cli.BoolFlag{ Name: "dry-run", Usage: "Don't perform any changes; instead log what would have been done", @@ -110,6 +123,7 @@ func main() { Value: "30s", Usage: `The minimum amount of seconds after which it is viable to join an existing cluster in which the current node is an expected member`, + EnvVar: "ETCDCD_MIN_UPTIME_TO_JOIN", }, } app.Action = func(c *cli.Context) { @@ -147,24 +161,30 @@ func parseArgs(c *cli.Context) *discovery.Discovery { if len(masterFilter) == 0 { log.Fatalf("'%s' is required", "master-names") } - minUptimeString := c.String("minimum-uptime-to-join") - minJoinUptime, err := time.ParseDuration(c.String("minimum-uptime-to-join")) - if err != nil { - log.Fatalf("Invalid duration '%s': %v", minUptimeString, err) - } return &discovery.Discovery{ - Platform: platform, - ConfigFile: c.String("platform-config-file"), - ClientPort: c.Int("client-port"), - ServerPort: c.Int("server-port"), - ClientScheme: c.String("client-scheme"), - ServerScheme: c.String("server-scheme"), - ProxyMode: c.Bool("proxy"), - IgnoreNamingMismatch: c.Bool("ignore-naming-mismatch"), - MasterFilter: masterFilter, - MaxTries: 5, - MinimumUptimeToJoin: minJoinUptime, + Platform: platform, + ConfigFile: c.String("platform-config-file"), + ClientPort: c.Int("client-port"), + ServerPort: c.Int("server-port"), + ClientScheme: c.String("client-scheme"), + ServerScheme: c.String("server-scheme"), + ProxyMode: c.Bool("proxy"), + IgnoreNamingMismatch: c.Bool("ignore-naming-mismatch"), + MasterFilter: masterFilter, + MasterCount: c.Int("master-count"), + MaxTries: 5, + MinUptimeToJoin: parseDuration(c, "minimum-uptime-to-join"), + MaxWaitForExpectedMasters: parseDuration(c, "maximum-master-resolution-time"), } } + +func parseDuration(c *cli.Context, flag string) time.Duration { + stringValue := c.String(flag) + duration, err := time.ParseDuration(stringValue) + if err != nil { + log.Fatalf("Invalid duration for '%s': '%s': %v", flag, stringValue, err) + } + return duration +} diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index d39efb3..18aa8c1 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -20,18 +20,20 @@ import ( // Discovery provides correct startup details for etcd with respect to // known vs. expected cluster membership type Discovery struct { - ConfigFile string - Platform string - ClientPort int - ServerPort int - ClientScheme string - ServerScheme string - MaxTries int - ProxyMode bool - MasterFilter string - DryRun bool - IgnoreNamingMismatch bool - MinimumUptimeToJoin time.Duration + ConfigFile string + Platform string + ClientPort int + ServerPort int + ClientScheme string + ServerScheme string + MaxTries int + ProxyMode bool + MasterFilter string + MasterCount int + DryRun bool + IgnoreNamingMismatch bool + MinUptimeToJoin time.Duration + MaxWaitForExpectedMasters time.Duration } func findMemberByName(members []etcd.Member, name string) *etcd.Member { @@ -67,8 +69,10 @@ func (d *Discovery) DiscoverEnvironment() (map[string]string, error) { return nil, errors.New("No such platform: " + d.Platform) } + timeout := time.Now().Add(d.MaxWaitForExpectedMasters) var expectedMembers []etcd.Member - for tries := 0; tries < d.MaxTries && len(expectedMembers) == 0; tries++ { + for len(expectedMembers) < d.MasterCount && time.Now().Before(timeout) { + expectedMembers = []etcd.Member{} if members, err := p.ExpectedMembers(d.MasterFilter, d.ClientScheme, d.ClientPort, d.ServerScheme, d.ServerPort); err == nil { for _, m := range members { @@ -81,7 +85,7 @@ func (d *Discovery) DiscoverEnvironment() (map[string]string, error) { } else { return nil, err } - if len(expectedMembers) == 0 { + if len(expectedMembers) < d.MasterCount { sleepTime := (2 * time.Second) if log.GetLevel() >= log.DebugLevel { log.Debugf("Failed to resolve expected members; sleeping for %s", sleepTime) @@ -109,7 +113,7 @@ func (d *Discovery) DiscoverEnvironment() (map[string]string, error) { log.Debugf("Local master: %#v", *localMaster) } // this instance is an expected master - if len(currentMembers) > 0 && uptime >= d.MinimumUptimeToJoin { + if len(currentMembers) > 0 && uptime >= d.MinUptimeToJoin { // there is an existing cluster if err = d.assertSaneClusterState(expectedMembers, currentMembers); err != nil { log.Fatal(err)