Merge pull request #153 from chizhg/kubetest2-gke-options

Segment the kubetest2 gke deployer flags to different groups and some other updates

kubetest2-gke/deployer/build.go

@@ -72,11 +72,11 @@ func (d *Deployer) Build() error {
 	// append the kubetest2 run id
 	// avoid double + in the version
 	// so they are valid docker tags
-	if !strings.HasSuffix(version, d.kubetest2CommonOptions.RunID()) {
+	if !strings.HasSuffix(version, d.Kubetest2CommonOptions.RunID()) {
 		if strings.Contains(version, "+") {
-			version += "-" + d.kubetest2CommonOptions.RunID()
+			version += "-" + d.Kubetest2CommonOptions.RunID()
 		} else {
-			version += "+" + d.kubetest2CommonOptions.RunID()
+			version += "+" + d.Kubetest2CommonOptions.RunID()
 		}
 	}
 
@@ -87,7 +87,7 @@ func (d *Deployer) Build() error {
 		}
 	}
 	d.ClusterVersion = version
-	build.StoreCommonBinaries(d.RepoRoot, d.kubetest2CommonOptions.RunDir())
+	build.StoreCommonBinaries(d.RepoRoot, d.Kubetest2CommonOptions.RunDir())
 	return nil
 }
 
@@ -96,13 +96,13 @@ func (d *Deployer) VerifyBuildFlags() error {
 		return fmt.Errorf("required repo-root when building from source")
 	}
 	d.BuildOptions.CommonBuildOptions.RepoRoot = d.RepoRoot
-	if d.kubetest2CommonOptions.ShouldBuild() && d.kubetest2CommonOptions.ShouldUp() && d.BuildOptions.CommonBuildOptions.StageLocation == "" {
+	if d.Kubetest2CommonOptions.ShouldBuild() && d.Kubetest2CommonOptions.ShouldUp() && d.BuildOptions.CommonBuildOptions.StageLocation == "" {
 		return fmt.Errorf("creating a gke cluster from built sources requires staging them to a specific GCS bucket, use --stage=gs://<bucket>")
 	}
 	// force extra GCP files to be staged
 	d.BuildOptions.CommonBuildOptions.StageExtraGCPFiles = true
 	// add kubetest2 runid as the version suffix
-	d.BuildOptions.CommonBuildOptions.VersionSuffix = d.kubetest2CommonOptions.RunID()
+	d.BuildOptions.CommonBuildOptions.VersionSuffix = d.Kubetest2CommonOptions.RunID()
 	return d.BuildOptions.Validate()
 }
 

kubetest2-gke/deployer/common.go

@@ -48,7 +48,7 @@ func (d *Deployer) Initialize() error {
 		klog.Warningf("--version is deprecated please use --cluster-version")
 		d.ClusterVersion = d.LegacyClusterVersion
 	}
-	if d.kubetest2CommonOptions.ShouldUp() {
+	if d.Kubetest2CommonOptions.ShouldUp() {
 		d.totalTryCount = math.Max(len(d.Regions), len(d.Zones))
 
 		if err := d.VerifyUpFlags(); err != nil {
@@ -74,46 +74,53 @@ func (d *Deployer) Initialize() error {
 			}
 			d.boskos = boskosClient
 
-			for i := 0; i < d.BoskosProjectsRequested; i++ {
-				resource, err := boskos.Acquire(
-					d.boskos,
-					d.BoskosResourceType,
-					time.Duration(d.BoskosAcquireTimeoutSeconds)*time.Second,
-					time.Duration(d.BoskosHeartbeatIntervalSeconds)*time.Second,
-					d.boskosHeartbeatClose,
-				)
-
-				if err != nil {
-					return fmt.Errorf("init failed to get project from boskos: %w", err)
+			for i := 0; i < len(d.BoskosProjectsRequested); i++ {
+				for j := 0; j < d.BoskosProjectsRequested[i]; j++ {
+					resource, err := boskos.Acquire(
+						d.boskos,
+						d.BoskosResourceType[i],
+						time.Duration(d.BoskosAcquireTimeoutSeconds)*time.Second,
+						time.Duration(d.BoskosHeartbeatIntervalSeconds)*time.Second,
+						d.boskosHeartbeatClose,
+					)
+
+					if err != nil {
+						return fmt.Errorf("init failed to get project from boskos: %w", err)
+					}
+					d.Projects = append(d.Projects, resource.Name)
+					klog.V(1).Infof("Got project %s from boskos", resource.Name)
 				}
-				d.Projects = append(d.Projects, resource.Name)
-				klog.V(1).Infof("Got project %s from boskos", resource.Name)
 			}
 		}
-
-		// Multi-cluster name adjustment
-		numProjects := len(d.Projects)
-		d.projectClustersLayout = make(map[string][]cluster, numProjects)
-		if numProjects > 1 {
-			if err := buildProjectClustersLayout(d.Projects, d.Clusters, d.projectClustersLayout); err != nil {
-				return fmt.Errorf("failed to build the project clusters layout: %v", err)
-			}
-		} else {
-			// Backwards compatible construction
-			clusters := make([]cluster, len(d.Clusters))
-			for i, clusterName := range d.Clusters {
-				clusters[i] = cluster{i, clusterName}
-			}
-			d.projectClustersLayout[d.Projects[0]] = clusters
-		}
 	}
 
-	if d.kubetest2CommonOptions.ShouldDown() {
+	if d.Kubetest2CommonOptions.ShouldDown() {
 		if err := d.VerifyDownFlags(); err != nil {
 			return fmt.Errorf("init failed to verify flags for down: %w", err)
 		}
 	}
 
+	// Multi-cluster name adjustment
+	numProjects := len(d.Projects)
+	d.projectClustersLayout = make(map[string][]cluster, numProjects)
+	if numProjects > 1 {
+		if err := buildProjectClustersLayout(d.Projects, d.Clusters, d.projectClustersLayout); err != nil {
+			return fmt.Errorf("failed to build the project clusters layout: %v", err)
+		}
+	} else {
+		// Backwards compatible construction
+		clusters := make([]cluster, len(d.Clusters))
+		for i, clusterName := range d.Clusters {
+			clusters[i] = cluster{i, clusterName}
+		}
+		d.projectClustersLayout[d.Projects[0]] = clusters
+	}
+
+	// Prepare the GCP environment for the following operations.
+	if err := d.PrepareGcpIfNeeded(d.Projects[0]); err != nil {
+		return err
+	}
+
 	return nil
 }
 

kubetest2-gke/deployer/deployer.go

@@ -101,11 +101,13 @@ type cluster struct {
 
 type Deployer struct {
 	// generic parts
-	kubetest2CommonOptions types.Options
+	Kubetest2CommonOptions types.Options
 
 	*options.BuildOptions
 	*options.CommonOptions
-	*options.UpOptions
+	*options.ProjectOptions
+	*options.NetworkOptions
+	*options.ClusterOptions
 
 	// doInit helps to make sure the initialization is performed only once
 	doInit sync.Once
@@ -127,6 +129,9 @@ type Deployer struct {
 	subnetworkRangesInternal             [][]string
 	privateClusterMasterIPRangesInternal [][]string
 
+	// the total number of Boskos projects to request
+	totalBoskosProjectsRequested int
+
 	// boskos struct field will be non-nil when the deployer is
 	// using boskos to acquire a GCP project
 	boskos *client.Client
@@ -152,9 +157,21 @@ func (d *Deployer) Version() string {
 
 // New implements deployer.New for gke
 func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
-	// create a deployer object and set fields that are not flag controlled
+	d := NewDeployer(opts)
+
+	// register flags
+	fs := bindFlags(d)
+
+	// register flags for klog
+	klog.InitFlags(nil)
+	fs.AddGoFlagSet(flag.CommandLine)
+	return d, fs
+}
+
+// NewDeployer returns a deployer object with fields that are not flag controlled
+func NewDeployer(opts types.Options) *Deployer {
 	d := &Deployer{
-		kubetest2CommonOptions: opts,
+		Kubetest2CommonOptions: opts,
 		BuildOptions: &options.BuildOptions{
 			CommonBuildOptions: &build.Options{
 				Builder:  &build.NoopBuilder{},
@@ -163,39 +180,37 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
 			},
 		},
 		CommonOptions: &options.CommonOptions{
-			Network:     "default",
-			Environment: "prod",
-		},
-		UpOptions: &options.UpOptions{
-			NumClusters:        1,
-			NumNodes:           defaultNodePool.Nodes,
-			MachineType:        defaultNodePool.MachineType,
-			ImageType:          defaultImage,
-			WindowsNumNodes:    defaultWindowsNodePool.Nodes,
-			WindowsMachineType: defaultWindowsNodePool.MachineType,
-			WindowsImageType:   defaultWindowsImage,
-			// Leave ClusterVersion as empty to use the default cluster version.
-			ClusterVersion:   "",
 			GCPSSHKeyIgnored: true,
-
+		},
+		ProjectOptions: &options.ProjectOptions{
 			BoskosLocation:                 defaultBoskosLocation,
-			BoskosResourceType:             defaultGKEProjectResourceType,
+			BoskosResourceType:             []string{defaultGKEProjectResourceType},
 			BoskosAcquireTimeoutSeconds:    defaultBoskosAcquireTimeoutSeconds,
 			BoskosHeartbeatIntervalSeconds: defaultBoskosHeartbeatIntervalSeconds,
-			BoskosProjectsRequested:        1,
+			BoskosProjectsRequested:        []int{1},
+		},
+		NetworkOptions: &options.NetworkOptions{
+			Network: "default",
+		},
+		ClusterOptions: &options.ClusterOptions{
+			Environment: "prod",
+			NumClusters: 1,
+			NumNodes:    defaultNodePool.Nodes,
+			MachineType: defaultNodePool.MachineType,
+			ImageType:   defaultImage,
+			// Leave ClusterVersion as empty to use the default cluster version.
+			ClusterVersion: "",
+
+			WindowsNumNodes:    defaultWindowsNodePool.Nodes,
+			WindowsMachineType: defaultWindowsNodePool.MachineType,
+			WindowsImageType:   defaultWindowsImage,
 
 			RetryableErrorPatterns: []string{gceStockoutErrorPattern},
 		},
 		localLogsDir: filepath.Join(opts.RunDir(), "logs"),
 	}
 
-	// register flags
-	fs := bindFlags(d)
-
-	// register flags for klog
-	klog.InitFlags(nil)
-	fs.AddGoFlagSet(flag.CommandLine)
-	return d, fs
+	return d
 }
 
 func (d *Deployer) VerifyLocationFlags() error {

kubetest2-gke/deployer/down.go

@@ -30,29 +30,23 @@ func (d *Deployer) Down() error {
 		return err
 	}
 
-	if len(d.Projects) > 0 {
-		if err := d.PrepareGcpIfNeeded(d.Projects[0]); err != nil {
-			return err
-		}
-
-		d.DeleteClusters(d.retryCount)
+	d.DeleteClusters(d.retryCount)
 
-		numDeletedFWRules, errCleanFirewalls := d.CleanupNetworkFirewalls(d.Projects[0], d.Network)
-		if errCleanFirewalls != nil {
-			klog.Errorf("Error cleaning-up firewall rules: %v", errCleanFirewalls)
-		} else {
-			klog.V(1).Infof("Deleted %d network firewall rules", numDeletedFWRules)
-		}
+	numDeletedFWRules, errCleanFirewalls := d.CleanupNetworkFirewalls(d.Projects[0], d.Network)
+	if errCleanFirewalls != nil {
+		klog.Errorf("Error cleaning-up firewall rules: %v", errCleanFirewalls)
+	} else {
+		klog.V(1).Infof("Deleted %d network firewall rules", numDeletedFWRules)
+	}
 
-		if err := d.TeardownNetwork(); err != nil {
-			return err
-		}
-		if err := d.DeleteSubnets(d.retryCount); err != nil {
-			return err
-		}
-		if err := d.DeleteNetwork(); err != nil {
-			return err
-		}
+	if err := d.TeardownNetwork(); err != nil {
+		return err
+	}
+	if err := d.DeleteSubnets(d.retryCount); err != nil {
+		return err
+	}
+	if err := d.DeleteNetwork(); err != nil {
+		return err
 	}
 
 	return nil

kubetest2-gke/deployer/network.go

@@ -57,7 +57,7 @@ func (d *Deployer) VerifyNetworkFlags() error {
 
 	numProjects := len(d.Projects)
 	if numProjects == 0 {
-		numProjects = d.BoskosProjectsRequested
+		numProjects = d.totalBoskosProjectsRequested
 	}
 
 	// Verify for multi-project profile.

kubetest2-gke/deployer/options/cluster.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import "fmt"
+
+type ClusterOptions struct {
+	Environment string `flag:"~environment" desc:"Container API endpoint to use, one of 'test', 'staging', 'prod', or a custom https:// URL. Defaults to prod if not provided"`
+
+	GcloudCommandGroup string `flag:"~gcloud-command-group" desc:"gcloud command group, can be one of empty, alpha, beta."`
+	Autopilot          bool   `flag:"~autopilot" desc:"Whether to create GKE Autopilot clusters or not."`
+	GcloudExtraFlags   string `flag:"~gcloud-extra-flags" desc:"Extra gcloud flags to pass when creating the clusters."`
+	CreateCommandFlag  string `flag:"~create-command" desc:"gcloud subcommand and additional flags used to create a cluster, such as container clusters create --quiet. If it's specified, --gcloud-command-group, --autopilot, --gcloud-extra-flags will be ignored."`
+
+	Regions []string `flag:"~region" desc:"Comma separated list for use with gcloud commands to specify the cluster region(s). The first region will be considered the primary region, and the rest will be considered the backup regions."`
+	Zones   []string `flag:"~zone" desc:"Comma separated list for use with gcloud commands to specify the cluster zone(s). The first zone will be considered the primary zone, and the rest will be considered the backup zones."`
+
+	NumClusters             int      `flag:"~num-clusters" desc:"Number of clusters to create, will auto-generate names as (kt2-<run-id>-<index>)."`
+	Clusters                []string `flag:"~cluster-name" desc:"Cluster names separated by comma. Must be set. For multi-project profile, it should be in the format of clusterA:0,clusterB:1,clusterC:2, where the index means the index of the project."`
+	MachineType             string   `flag:"~machine-type" desc:"For use with gcloud commands to specify the machine type for the cluster."`
+	NumNodes                int      `flag:"~num-nodes" desc:"For use with gcloud commands to specify the number of nodes for the cluster."`
+	ImageType               string   `flag:"~image-type" desc:"The image type to use for the cluster."`
+	ReleaseChannel          string   `desc:"Use a GKE release channel, could be one of empty, rapid, regular and stable - https://cloud.google.com/kubernetes-engine/docs/concepts/release-channels"`
+	LegacyClusterVersion    string   `flag:"~version,deprecated" desc:"Use --cluster-version instead"`
+	ClusterVersion          string   `desc:"Use a specific GKE version e.g. 1.16.13.gke-400, 'latest' or ''. If --build is specified it will default to building kubernetes from source."`
+	WorkloadIdentityEnabled bool     `flag:"~enable-workload-identity" desc:"Whether enable workload identity for the cluster or not. See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity."`
+
+	WindowsEnabled     bool   `flag:"~enable-windows" desc:"Whether enable Windows node pool in the cluster or not."`
+	WindowsNumNodes    int    `flag:"~windows-num-nodes" desc:"For use with gcloud commands to specify the number of nodes for Windows node pools in the cluster."`
+	WindowsMachineType string `flag:"~windows-machine-type" desc:"For use with gcloud commands to specify the machine type for Windows node in the cluster."`
+	WindowsImageType   string `flag:"~windows-image-type" desc:"The Windows image type to use for the cluster."`
+
+	RetryableErrorPatterns []string `flag:"~retryable-error-patterns" desc:"Comma separated list of regex match patterns for retryable errors during cluster creation."`
+}
+
+func (uo *ClusterOptions) Validate() error {
+	// allow max 99 clusters (should be sufficient for most use cases)
+	if uo.NumClusters < 1 || uo.NumClusters > 99 {
+		return fmt.Errorf("need to specify between 1 and 99 clusters got %q: ", uo.NumClusters)
+	}
+	return nil
+}

kubetest2-gke/deployer/options/common.go

@@ -17,13 +17,7 @@ limitations under the License.
 package options
 
 type CommonOptions struct {
-	RepoRoot string `desc:"Path to root of the kubernetes repo. Used with --build and for dumping cluster logs."`
-
-	Environment       string   `flag:"~environment" desc:"Container API endpoint to use, one of 'test', 'staging', 'prod', or a custom https:// URL. Defaults to prod if not provided"`
-	Projects          []string `flag:"~project" desc:"Comma separated list of GCP Project(s) to use for creating the cluster."`
-	Clusters          []string `flag:"~cluster-name" desc:"Cluster names separated by comma. Must be set. For multi-project profile, it should be in the format of clusterA:0,clusterB:1,clusterC:2, where the index means the index of the project."`
-	Regions           []string `flag:"~region" desc:"Comma separated list for use with gcloud commands to specify the cluster region(s). The first region will be considered the primary region, and the rest will be considered the backup regions."`
-	Zones             []string `flag:"~zone" desc:"Comma separated list for use with gcloud commands to specify the cluster zone(s). The first zone will be considered the primary zone, and the rest will be considered the backup zones."`
-	Network           string   `flag:"~network" desc:"Cluster network. Defaults to the default network if not provided. For multi-project use cases, this will be the Shared VPC network name."`
-	GCPServiceAccount string   `flag:"~gcp-service-account" desc:"Service account to activate before using gcloud."`
+	RepoRoot          string `desc:"Path to root of the kubernetes repo. Used with --build and for dumping cluster logs."`
+	GCPServiceAccount string `flag:"~gcp-service-account" desc:"Service account to activate before using gcloud."`
+	GCPSSHKeyIgnored  bool   `flag:"~ignore-gcp-ssh-key" desc:"Whether the GCP SSH key should be ignored or not for bringing up the cluster."`
 }

kubetest2-gke/deployer/options/network.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+type NetworkOptions struct {
+	Network string `flag:"~network" desc:"Cluster network. Defaults to the default network if not provided. For multi-project use cases, this will be the Shared VPC network name."`
+
+	PrivateClusterAccessLevel    string   `flag:"~private-cluster-access-level" desc:"Private cluster access level, if not empty, must be one of 'no', 'limited' or 'unrestricted'. See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters."`
+	PrivateClusterMasterIPRanges []string `flag:"~private-cluster-master-ip-range" desc:"Private cluster master IP ranges. It should be IPv4 CIDR(s), and its length must be the same as the number of clusters if private cluster is requested."`
+	SubnetworkRanges             []string `flag:"~subnetwork-ranges" desc:"Subnetwork ranges as required for shared VPC setup as described in https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc#creating_a_network_and_two_subnets. For multi-project profile, it is required and should be in the format of 10.0.4.0/22 10.0.32.0/20 10.4.0.0/14,172.16.4.0/22 172.16.16.0/20 172.16.4.0/22, where the subnetworks configuration for different project are separated by comma, and the ranges of each subnetwork configuration is separated by space."`
+}