add max wait time & expected count for masters

matt-deboer · Apr 3, 2017 · 5d32134 · 5d32134
1 parent 5251ed3
commit 5d32134
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 31 deletions.
diff --git a/pkg/cmd/main.go b/pkg/cmd/main.go
@@ -89,6 +89,19 @@ func main() {
 				of the masters (e.g., 'k8s-master-*')`,
 			EnvVar: "ETCDCD_MASTER_NAMES",
 		},
+		cli.IntFlag{
+			Name:   "master-count",
+			Value:  3,
+			Usage:  `The expected number of masters in an intial cluster; ignored when joining an existing cluster`,
+			EnvVar: "ETCDCD_MASTER_COUNT",
+		},
+		cli.StringFlag{
+			Name:  "maximum-master-resolution-time",
+			Value: "5m",
+			Usage: `The minimum amount of seconds after which it is viable to join an existing cluster in which
+				the current node is an expected member`,
+			EnvVar: "ETCDCD_MAX_MASTER_RESOLUTION_TIME",
+		},
 		cli.BoolFlag{
 			Name:   "dry-run",
 			Usage:  "Don't perform any changes; instead log what would have been done",
@@ -110,6 +123,7 @@ func main() {
 			Value: "30s",
 			Usage: `The minimum amount of seconds after which it is viable to join an existing cluster in which
 				the current node is an expected member`,
+			EnvVar: "ETCDCD_MIN_UPTIME_TO_JOIN",
 		},
 	}
 	app.Action = func(c *cli.Context) {
@@ -147,24 +161,30 @@ func parseArgs(c *cli.Context) *discovery.Discovery {
 	if len(masterFilter) == 0 {
 		log.Fatalf("'%s' is required", "master-names")
 	}
-	minUptimeString := c.String("minimum-uptime-to-join")
-	minJoinUptime, err := time.ParseDuration(c.String("minimum-uptime-to-join"))
-	if err != nil {
-		log.Fatalf("Invalid duration '%s': %v", minUptimeString, err)
-	}
 
 	return &discovery.Discovery{
-		Platform:             platform,
-		ConfigFile:           c.String("platform-config-file"),
-		ClientPort:           c.Int("client-port"),
-		ServerPort:           c.Int("server-port"),
-		ClientScheme:         c.String("client-scheme"),
-		ServerScheme:         c.String("server-scheme"),
-		ProxyMode:            c.Bool("proxy"),
-		IgnoreNamingMismatch: c.Bool("ignore-naming-mismatch"),
-		MasterFilter:         masterFilter,
-		MaxTries:             5,
-		MinimumUptimeToJoin:  minJoinUptime,
+		Platform:                  platform,
+		ConfigFile:                c.String("platform-config-file"),
+		ClientPort:                c.Int("client-port"),
+		ServerPort:                c.Int("server-port"),
+		ClientScheme:              c.String("client-scheme"),
+		ServerScheme:              c.String("server-scheme"),
+		ProxyMode:                 c.Bool("proxy"),
+		IgnoreNamingMismatch:      c.Bool("ignore-naming-mismatch"),
+		MasterFilter:              masterFilter,
+		MasterCount:               c.Int("master-count"),
+		MaxTries:                  5,
+		MinUptimeToJoin:           parseDuration(c, "minimum-uptime-to-join"),
+		MaxWaitForExpectedMasters: parseDuration(c, "maximum-master-resolution-time"),
 	}
 
 }
+
+func parseDuration(c *cli.Context, flag string) time.Duration {
+	stringValue := c.String(flag)
+	duration, err := time.ParseDuration(stringValue)
+	if err != nil {
+		log.Fatalf("Invalid duration for '%s': '%s': %v", flag, stringValue, err)
+	}
+	return duration
+}
diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go
@@ -20,18 +20,20 @@ import (
 // Discovery provides correct startup details for etcd with respect to
 // known vs. expected cluster membership
 type Discovery struct {
-	ConfigFile           string
-	Platform             string
-	ClientPort           int
-	ServerPort           int
-	ClientScheme         string
-	ServerScheme         string
-	MaxTries             int
-	ProxyMode            bool
-	MasterFilter         string
-	DryRun               bool
-	IgnoreNamingMismatch bool
-	MinimumUptimeToJoin  time.Duration
+	ConfigFile                string
+	Platform                  string
+	ClientPort                int
+	ServerPort                int
+	ClientScheme              string
+	ServerScheme              string
+	MaxTries                  int
+	ProxyMode                 bool
+	MasterFilter              string
+	MasterCount               int
+	DryRun                    bool
+	IgnoreNamingMismatch      bool
+	MinUptimeToJoin           time.Duration
+	MaxWaitForExpectedMasters time.Duration
 }
 
 func findMemberByName(members []etcd.Member, name string) *etcd.Member {
@@ -67,8 +69,10 @@ func (d *Discovery) DiscoverEnvironment() (map[string]string, error) {
 		return nil, errors.New("No such platform: " + d.Platform)
 	}
 
+	timeout := time.Now().Add(d.MaxWaitForExpectedMasters)
 	var expectedMembers []etcd.Member
-	for tries := 0; tries < d.MaxTries && len(expectedMembers) == 0; tries++ {
+	for len(expectedMembers) < d.MasterCount && time.Now().Before(timeout) {
+		expectedMembers = []etcd.Member{}
 		if members, err := p.ExpectedMembers(d.MasterFilter, d.ClientScheme,
 			d.ClientPort, d.ServerScheme, d.ServerPort); err == nil {
 			for _, m := range members {
@@ -81,7 +85,7 @@ func (d *Discovery) DiscoverEnvironment() (map[string]string, error) {
 		} else {
 			return nil, err
 		}
-		if len(expectedMembers) == 0 {
+		if len(expectedMembers) < d.MasterCount {
 			sleepTime := (2 * time.Second)
 			if log.GetLevel() >= log.DebugLevel {
 				log.Debugf("Failed to resolve expected members; sleeping for %s", sleepTime)
@@ -109,7 +113,7 @@ func (d *Discovery) DiscoverEnvironment() (map[string]string, error) {
 			log.Debugf("Local master: %#v", *localMaster)
 		}
 		// this instance is an expected master
-		if len(currentMembers) > 0 && uptime >= d.MinimumUptimeToJoin {
+		if len(currentMembers) > 0 && uptime >= d.MinUptimeToJoin {
 			// there is an existing cluster
 			if err = d.assertSaneClusterState(expectedMembers, currentMembers); err != nil {
 				log.Fatal(err)