From 2163b1ea185ac4abbcb312a8b61b51f92152862d Mon Sep 17 00:00:00 2001 From: Chi Zhang Date: Sun, 20 Jun 2021 19:31:56 -0700 Subject: [PATCH 1/4] Reorganize the kubetest2-gke deployer flags --- kubetest2-gke/deployer/deployer.go | 36 ++++++++------ kubetest2-gke/deployer/options/cluster.go | 56 +++++++++++++++++++++ kubetest2-gke/deployer/options/common.go | 12 ++--- kubetest2-gke/deployer/options/network.go | 25 ++++++++++ kubetest2-gke/deployer/options/project.go | 27 ++++++++++ kubetest2-gke/deployer/options/up.go | 60 ----------------------- kubetest2-gke/deployer/up.go | 4 +- 7 files changed, 134 insertions(+), 86 deletions(-) create mode 100644 kubetest2-gke/deployer/options/cluster.go create mode 100644 kubetest2-gke/deployer/options/network.go create mode 100644 kubetest2-gke/deployer/options/project.go delete mode 100644 kubetest2-gke/deployer/options/up.go diff --git a/kubetest2-gke/deployer/deployer.go b/kubetest2-gke/deployer/deployer.go index 36d84d0d..bbc98b25 100644 --- a/kubetest2-gke/deployer/deployer.go +++ b/kubetest2-gke/deployer/deployer.go @@ -105,7 +105,9 @@ type Deployer struct { *options.BuildOptions *options.CommonOptions - *options.UpOptions + *options.ProjectOptions + *options.NetworkOptions + *options.ClusterOptions // doInit helps to make sure the initialization is performed only once doInit sync.Once @@ -163,26 +165,30 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { }, }, CommonOptions: &options.CommonOptions{ - Network: "default", - Environment: "prod", - }, - UpOptions: &options.UpOptions{ - NumClusters: 1, - NumNodes: defaultNodePool.Nodes, - MachineType: defaultNodePool.MachineType, - ImageType: defaultImage, - WindowsNumNodes: defaultWindowsNodePool.Nodes, - WindowsMachineType: defaultWindowsNodePool.MachineType, - WindowsImageType: defaultWindowsImage, - // Leave ClusterVersion as empty to use the default cluster version. - ClusterVersion: "", GCPSSHKeyIgnored: true, - + }, + ProjectOptions: &options.ProjectOptions{ BoskosLocation: defaultBoskosLocation, BoskosResourceType: defaultGKEProjectResourceType, BoskosAcquireTimeoutSeconds: defaultBoskosAcquireTimeoutSeconds, BoskosHeartbeatIntervalSeconds: defaultBoskosHeartbeatIntervalSeconds, BoskosProjectsRequested: 1, + }, + NetworkOptions: &options.NetworkOptions{ + Network: "default", + }, + ClusterOptions: &options.ClusterOptions{ + Environment: "prod", + NumClusters: 1, + NumNodes: defaultNodePool.Nodes, + MachineType: defaultNodePool.MachineType, + ImageType: defaultImage, + // Leave ClusterVersion as empty to use the default cluster version. + ClusterVersion: "", + + WindowsNumNodes: defaultWindowsNodePool.Nodes, + WindowsMachineType: defaultWindowsNodePool.MachineType, + WindowsImageType: defaultWindowsImage, RetryableErrorPatterns: []string{gceStockoutErrorPattern}, }, diff --git a/kubetest2-gke/deployer/options/cluster.go b/kubetest2-gke/deployer/options/cluster.go new file mode 100644 index 00000000..d663891a --- /dev/null +++ b/kubetest2-gke/deployer/options/cluster.go @@ -0,0 +1,56 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import "fmt" + +type ClusterOptions struct { + Environment string `flag:"~environment" desc:"Container API endpoint to use, one of 'test', 'staging', 'prod', or a custom https:// URL. Defaults to prod if not provided"` + + GcloudCommandGroup string `flag:"~gcloud-command-group" desc:"gcloud command group, can be one of empty, alpha, beta."` + Autopilot bool `flag:"~autopilot" desc:"Whether to create GKE Autopilot clusters or not."` + GcloudExtraFlags string `flag:"~gcloud-extra-flags" desc:"Extra gcloud flags to pass when creating the clusters."` + CreateCommandFlag string `flag:"~create-command" desc:"gcloud subcommand and additional flags used to create a cluster, such as container clusters create --quiet. If it's specified, --gcloud-command-group, --autopilot, --gcloud-extra-flags will be ignored."` + + Regions []string `flag:"~region" desc:"Comma separated list for use with gcloud commands to specify the cluster region(s). The first region will be considered the primary region, and the rest will be considered the backup regions."` + Zones []string `flag:"~zone" desc:"Comma separated list for use with gcloud commands to specify the cluster zone(s). The first zone will be considered the primary zone, and the rest will be considered the backup zones."` + + NumClusters int `flag:"~num-clusters" desc:"Number of clusters to create, will auto-generate names as (kt2--)."` + Clusters []string `flag:"~cluster-name" desc:"Cluster names separated by comma. Must be set. For multi-project profile, it should be in the format of clusterA:0,clusterB:1,clusterC:2, where the index means the index of the project."` + MachineType string `flag:"~machine-type" desc:"For use with gcloud commands to specify the machine type for the cluster."` + NumNodes int `flag:"~num-nodes" desc:"For use with gcloud commands to specify the number of nodes for the cluster."` + ImageType string `flag:"~image-type" desc:"The image type to use for the cluster."` + ReleaseChannel string `desc:"Use a GKE release channel, could be one of empty, rapid, regular and stable - https://cloud.google.com/kubernetes-engine/docs/concepts/release-channels"` + LegacyClusterVersion string `flag:"~version,deprecated" desc:"Use --cluster-version instead"` + ClusterVersion string `desc:"Use a specific GKE version e.g. 1.16.13.gke-400, 'latest' or ''. If --build is specified it will default to building kubernetes from source."` + WorkloadIdentityEnabled bool `flag:"~enable-workload-identity" desc:"Whether enable workload identity for the cluster or not. See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity."` + + WindowsEnabled bool `flag:"~enable-windows" desc:"Whether enable Windows node pool in the cluster or not."` + WindowsNumNodes int `flag:"~windows-num-nodes" desc:"For use with gcloud commands to specify the number of nodes for Windows node pools in the cluster."` + WindowsMachineType string `flag:"~windows-machine-type" desc:"For use with gcloud commands to specify the machine type for Windows node in the cluster."` + WindowsImageType string `flag:"~windows-image-type" desc:"The Windows image type to use for the cluster."` + + RetryableErrorPatterns []string `flag:"~retryable-error-patterns" desc:"Comma separated list of regex match patterns for retryable errors during cluster creation."` +} + +func (uo *ClusterOptions) Validate() error { + // allow max 99 clusters (should be sufficient for most use cases) + if uo.NumClusters < 1 || uo.NumClusters > 99 { + return fmt.Errorf("need to specify between 1 and 99 clusters got %q: ", uo.NumClusters) + } + return nil +} diff --git a/kubetest2-gke/deployer/options/common.go b/kubetest2-gke/deployer/options/common.go index 1e0ff879..4384fb68 100644 --- a/kubetest2-gke/deployer/options/common.go +++ b/kubetest2-gke/deployer/options/common.go @@ -17,13 +17,7 @@ limitations under the License. package options type CommonOptions struct { - RepoRoot string `desc:"Path to root of the kubernetes repo. Used with --build and for dumping cluster logs."` - - Environment string `flag:"~environment" desc:"Container API endpoint to use, one of 'test', 'staging', 'prod', or a custom https:// URL. Defaults to prod if not provided"` - Projects []string `flag:"~project" desc:"Comma separated list of GCP Project(s) to use for creating the cluster."` - Clusters []string `flag:"~cluster-name" desc:"Cluster names separated by comma. Must be set. For multi-project profile, it should be in the format of clusterA:0,clusterB:1,clusterC:2, where the index means the index of the project."` - Regions []string `flag:"~region" desc:"Comma separated list for use with gcloud commands to specify the cluster region(s). The first region will be considered the primary region, and the rest will be considered the backup regions."` - Zones []string `flag:"~zone" desc:"Comma separated list for use with gcloud commands to specify the cluster zone(s). The first zone will be considered the primary zone, and the rest will be considered the backup zones."` - Network string `flag:"~network" desc:"Cluster network. Defaults to the default network if not provided. For multi-project use cases, this will be the Shared VPC network name."` - GCPServiceAccount string `flag:"~gcp-service-account" desc:"Service account to activate before using gcloud."` + RepoRoot string `desc:"Path to root of the kubernetes repo. Used with --build and for dumping cluster logs."` + GCPServiceAccount string `flag:"~gcp-service-account" desc:"Service account to activate before using gcloud."` + GCPSSHKeyIgnored bool `flag:"~ignore-gcp-ssh-key" desc:"Whether the GCP SSH key should be ignored or not for bringing up the cluster."` } diff --git a/kubetest2-gke/deployer/options/network.go b/kubetest2-gke/deployer/options/network.go new file mode 100644 index 00000000..37994b22 --- /dev/null +++ b/kubetest2-gke/deployer/options/network.go @@ -0,0 +1,25 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +type NetworkOptions struct { + Network string `flag:"~network" desc:"Cluster network. Defaults to the default network if not provided. For multi-project use cases, this will be the Shared VPC network name."` + + PrivateClusterAccessLevel string `flag:"~private-cluster-access-level" desc:"Private cluster access level, if not empty, must be one of 'no', 'limited' or 'unrestricted'. See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters."` + PrivateClusterMasterIPRanges []string `flag:"~private-cluster-master-ip-range" desc:"Private cluster master IP ranges. It should be IPv4 CIDR(s), and its length must be the same as the number of clusters if private cluster is requested."` + SubnetworkRanges []string `flag:"~subnetwork-ranges" desc:"Subnetwork ranges as required for shared VPC setup as described in https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc#creating_a_network_and_two_subnets. For multi-project profile, it is required and should be in the format of 10.0.4.0/22 10.0.32.0/20 10.4.0.0/14,172.16.4.0/22 172.16.16.0/20 172.16.4.0/22, where the subnetworks configuration for different project are separated by comma, and the ranges of each subnetwork configuration is separated by space."` +} diff --git a/kubetest2-gke/deployer/options/project.go b/kubetest2-gke/deployer/options/project.go new file mode 100644 index 00000000..004aef1e --- /dev/null +++ b/kubetest2-gke/deployer/options/project.go @@ -0,0 +1,27 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +type ProjectOptions struct { + Projects []string `flag:"~project" desc:"Comma separated list of GCP Project(s) to use for creating the cluster."` + + BoskosLocation string `flag:"~boskos-location" desc:"If set, manually specifies the location of the Boskos server."` + BoskosResourceType string `flag:"~boskos-resource-type" desc:"If set, manually specifies the resource type of GCP projects to acquire from Boskos."` + BoskosAcquireTimeoutSeconds int `flag:"~boskos-acquire-timeout-seconds" desc:"How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring."` + BoskosHeartbeatIntervalSeconds int `flag:"~boskos-heartbeat-interval-seconds" desc:"How often (in seconds) to send a heartbeat to Boskos to hold the acquired resource. 0 means no heartbeat."` + BoskosProjectsRequested int `flag:"~projects-requested" desc:"Number of projects to request from Boskos. It is only respected if projects is empty, and must be larger than zero."` +} diff --git a/kubetest2-gke/deployer/options/up.go b/kubetest2-gke/deployer/options/up.go deleted file mode 100644 index 90c5e815..00000000 --- a/kubetest2-gke/deployer/options/up.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import "fmt" - -type UpOptions struct { - GcloudCommandGroup string `flag:"~gcloud-command-group" desc:"gcloud command group, can be one of empty, alpha, beta."` - Autopilot bool `flag:"~autopilot" desc:"Whether to create GKE Autopilot clusters or not."` - GcloudExtraFlags string `flag:"~gcloud-extra-flags" desc:"Extra gcloud flags to pass when creating the clusters."` - CreateCommandFlag string `flag:"~create-command" desc:"gcloud subcommand and additional flags used to create a cluster, such as container clusters create --quiet. If it's specified, --gcloud-command-group, --autopilot, --gcloud-extra-flags will be ignored."` - - NumClusters int `flag:"~num-clusters" desc:"Number of clusters to create, will auto-generate names as (kt2--)."` - MachineType string `flag:"~machine-type" desc:"For use with gcloud commands to specify the machine type for the cluster."` - NumNodes int `flag:"~num-nodes" desc:"For use with gcloud commands to specify the number of nodes for the cluster."` - ImageType string `flag:"~image-type" desc:"The image type to use for the cluster."` - ReleaseChannel string `desc:"Use a GKE release channel, could be one of empty, rapid, regular and stable - https://cloud.google.com/kubernetes-engine/docs/concepts/release-channels"` - ClusterVersion string `desc:"Use a specific GKE version e.g. 1.16.13.gke-400, 'latest' or ''. If --build is specified it will default to building kubernetes from source."` - LegacyClusterVersion string `flag:"~version,deprecated" desc:"Use --cluster-version instead"` - WorkloadIdentityEnabled bool `flag:"~enable-workload-identity" desc:"Whether enable workload identity for the cluster or not. See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity."` - GCPSSHKeyIgnored bool `flag:"~ignore-gcp-ssh-key" desc:"Whether the GCP SSH key should be ignored or not for bringing up the cluster."` - WindowsEnabled bool `flag:"~enable-windows" desc:"Whether enable Windows node pool in the cluster or not."` - WindowsNumNodes int `flag:"~windows-num-nodes" desc:"For use with gcloud commands to specify the number of nodes for Windows node pools in the cluster."` - WindowsMachineType string `flag:"~windows-machine-type" desc:"For use with gcloud commands to specify the machine type for Windows node in the cluster."` - WindowsImageType string `flag:"~windows-image-type" desc:"The Windows image type to use for the cluster."` - - BoskosLocation string `flag:"~boskos-location" desc:"If set, manually specifies the location of the Boskos server."` - BoskosResourceType string `flag:"~boskos-resource-type" desc:"If set, manually specifies the resource type of GCP projects to acquire from Boskos."` - BoskosAcquireTimeoutSeconds int `flag:"~boskos-acquire-timeout-seconds" desc:"How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring."` - BoskosHeartbeatIntervalSeconds int `flag:"~boskos-heartbeat-interval-seconds" desc:"How often (in seconds) to send a heartbeat to Boskos to hold the acquired resource. 0 means no heartbeat."` - BoskosProjectsRequested int `flag:"~projects-requested" desc:"Number of projects to request from Boskos. It is only respected if projects is empty, and must be larger than zero."` - - PrivateClusterAccessLevel string `flag:"~private-cluster-access-level" desc:"Private cluster access level, if not empty, must be one of 'no', 'limited' or 'unrestricted'. See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters."` - PrivateClusterMasterIPRanges []string `flag:"~private-cluster-master-ip-range" desc:"Private cluster master IP ranges. It should be IPv4 CIDR(s), and its length must be the same as the number of clusters if private cluster is requested."` - SubnetworkRanges []string `flag:"~subnetwork-ranges" desc:"Subnetwork ranges as required for shared VPC setup as described in https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc#creating_a_network_and_two_subnets. For multi-project profile, it is required and should be in the format of 10.0.4.0/22 10.0.32.0/20 10.4.0.0/14,172.16.4.0/22 172.16.16.0/20 172.16.4.0/22, where the subnetworks configuration for different project are separated by comma, and the ranges of each subnetwork configuration is separated by space."` - - RetryableErrorPatterns []string `flag:"~retryable-error-patterns" desc:"Comma separated list of regex match patterns for retryable errors during cluster creation."` -} - -func (uo *UpOptions) Validate() error { - // allow max 99 clusters (should be sufficient for most use cases) - if uo.NumClusters < 1 || uo.NumClusters > 99 { - return fmt.Errorf("need to specify between 1 and 99 clusters got %q: ", uo.NumClusters) - } - return nil -} diff --git a/kubetest2-gke/deployer/up.go b/kubetest2-gke/deployer/up.go index e7e157e1..fa18022f 100644 --- a/kubetest2-gke/deployer/up.go +++ b/kubetest2-gke/deployer/up.go @@ -336,10 +336,10 @@ func (d *Deployer) VerifyUpFlags() error { if len(d.Projects) > 1 || d.BoskosProjectsRequested > 1 { return fmt.Errorf("explicit --cluster-name must be set for multi-project profile") } - if err := d.UpOptions.Validate(); err != nil { + if err := d.ClusterOptions.Validate(); err != nil { return err } - d.Clusters = generateClusterNames(d.UpOptions.NumClusters, d.kubetest2CommonOptions.RunID()) + d.Clusters = generateClusterNames(d.ClusterOptions.NumClusters, d.kubetest2CommonOptions.RunID()) } else { klog.V(0).Infof("explicit --cluster-name specified, ignoring --num-clusters") } From 517310a9fc1ade83789ea8057c2a69618d8269bc Mon Sep 17 00:00:00 2001 From: Chi Zhang Date: Sun, 18 Jul 2021 17:56:32 -0700 Subject: [PATCH 2/4] Segment the kubetest2 gke deployer flags to different groups --- kubetest2-gke/deployer/build.go | 12 ++++---- kubetest2-gke/deployer/common.go | 32 ++++++++++---------- kubetest2-gke/deployer/deployer.go | 33 +++++++++++++-------- kubetest2-gke/deployer/down.go | 36 ++++++++++------------- kubetest2-gke/deployer/network.go | 2 +- kubetest2-gke/deployer/options/project.go | 10 +++---- kubetest2-gke/deployer/up.go | 24 +++++++-------- 7 files changed, 77 insertions(+), 72 deletions(-) diff --git a/kubetest2-gke/deployer/build.go b/kubetest2-gke/deployer/build.go index 727e7a5a..a8955c26 100644 --- a/kubetest2-gke/deployer/build.go +++ b/kubetest2-gke/deployer/build.go @@ -72,11 +72,11 @@ func (d *Deployer) Build() error { // append the kubetest2 run id // avoid double + in the version // so they are valid docker tags - if !strings.HasSuffix(version, d.kubetest2CommonOptions.RunID()) { + if !strings.HasSuffix(version, d.Kubetest2CommonOptions.RunID()) { if strings.Contains(version, "+") { - version += "-" + d.kubetest2CommonOptions.RunID() + version += "-" + d.Kubetest2CommonOptions.RunID() } else { - version += "+" + d.kubetest2CommonOptions.RunID() + version += "+" + d.Kubetest2CommonOptions.RunID() } } @@ -87,7 +87,7 @@ func (d *Deployer) Build() error { } } d.ClusterVersion = version - build.StoreCommonBinaries(d.RepoRoot, d.kubetest2CommonOptions.RunDir()) + build.StoreCommonBinaries(d.RepoRoot, d.Kubetest2CommonOptions.RunDir()) return nil } @@ -96,13 +96,13 @@ func (d *Deployer) VerifyBuildFlags() error { return fmt.Errorf("required repo-root when building from source") } d.BuildOptions.CommonBuildOptions.RepoRoot = d.RepoRoot - if d.kubetest2CommonOptions.ShouldBuild() && d.kubetest2CommonOptions.ShouldUp() && d.BuildOptions.CommonBuildOptions.StageLocation == "" { + if d.Kubetest2CommonOptions.ShouldBuild() && d.Kubetest2CommonOptions.ShouldUp() && d.BuildOptions.CommonBuildOptions.StageLocation == "" { return fmt.Errorf("creating a gke cluster from built sources requires staging them to a specific GCS bucket, use --stage=gs://") } // force extra GCP files to be staged d.BuildOptions.CommonBuildOptions.StageExtraGCPFiles = true // add kubetest2 runid as the version suffix - d.BuildOptions.CommonBuildOptions.VersionSuffix = d.kubetest2CommonOptions.RunID() + d.BuildOptions.CommonBuildOptions.VersionSuffix = d.Kubetest2CommonOptions.RunID() return d.BuildOptions.Validate() } diff --git a/kubetest2-gke/deployer/common.go b/kubetest2-gke/deployer/common.go index b6e41fa2..cce582d9 100644 --- a/kubetest2-gke/deployer/common.go +++ b/kubetest2-gke/deployer/common.go @@ -48,7 +48,7 @@ func (d *Deployer) Initialize() error { klog.Warningf("--version is deprecated please use --cluster-version") d.ClusterVersion = d.LegacyClusterVersion } - if d.kubetest2CommonOptions.ShouldUp() { + if d.Kubetest2CommonOptions.ShouldUp() { d.totalTryCount = math.Max(len(d.Regions), len(d.Zones)) if err := d.VerifyUpFlags(); err != nil { @@ -74,20 +74,22 @@ func (d *Deployer) Initialize() error { } d.boskos = boskosClient - for i := 0; i < d.BoskosProjectsRequested; i++ { - resource, err := boskos.Acquire( - d.boskos, - d.BoskosResourceType, - time.Duration(d.BoskosAcquireTimeoutSeconds)*time.Second, - time.Duration(d.BoskosHeartbeatIntervalSeconds)*time.Second, - d.boskosHeartbeatClose, - ) - - if err != nil { - return fmt.Errorf("init failed to get project from boskos: %w", err) + for i := 0; i < len(d.BoskosProjectsRequested); i++ { + for j := 0; j < d.BoskosProjectsRequested[i]; j++ { + resource, err := boskos.Acquire( + d.boskos, + d.BoskosResourceType[i], + time.Duration(d.BoskosAcquireTimeoutSeconds)*time.Second, + time.Duration(d.BoskosHeartbeatIntervalSeconds)*time.Second, + d.boskosHeartbeatClose, + ) + + if err != nil { + return fmt.Errorf("init failed to get project from boskos: %w", err) + } + d.Projects = append(d.Projects, resource.Name) + klog.V(1).Infof("Got project %s from boskos", resource.Name) } - d.Projects = append(d.Projects, resource.Name) - klog.V(1).Infof("Got project %s from boskos", resource.Name) } } @@ -108,7 +110,7 @@ func (d *Deployer) Initialize() error { } } - if d.kubetest2CommonOptions.ShouldDown() { + if d.Kubetest2CommonOptions.ShouldDown() { if err := d.VerifyDownFlags(); err != nil { return fmt.Errorf("init failed to verify flags for down: %w", err) } diff --git a/kubetest2-gke/deployer/deployer.go b/kubetest2-gke/deployer/deployer.go index bbc98b25..0b9839f6 100644 --- a/kubetest2-gke/deployer/deployer.go +++ b/kubetest2-gke/deployer/deployer.go @@ -101,7 +101,7 @@ type cluster struct { type Deployer struct { // generic parts - kubetest2CommonOptions types.Options + Kubetest2CommonOptions types.Options *options.BuildOptions *options.CommonOptions @@ -129,6 +129,9 @@ type Deployer struct { subnetworkRangesInternal [][]string privateClusterMasterIPRangesInternal [][]string + // the total number of Boskos projects to request + totalBoskosProjectsRequested int + // boskos struct field will be non-nil when the deployer is // using boskos to acquire a GCP project boskos *client.Client @@ -154,9 +157,21 @@ func (d *Deployer) Version() string { // New implements deployer.New for gke func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { - // create a deployer object and set fields that are not flag controlled + d := NewDeployer(opts) + + // register flags + fs := bindFlags(d) + + // register flags for klog + klog.InitFlags(nil) + fs.AddGoFlagSet(flag.CommandLine) + return d, fs +} + +// NewDeployer returns a deployer object with fields that are not flag controlled +func NewDeployer(opts types.Options) *Deployer { d := &Deployer{ - kubetest2CommonOptions: opts, + Kubetest2CommonOptions: opts, BuildOptions: &options.BuildOptions{ CommonBuildOptions: &build.Options{ Builder: &build.NoopBuilder{}, @@ -169,10 +184,10 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { }, ProjectOptions: &options.ProjectOptions{ BoskosLocation: defaultBoskosLocation, - BoskosResourceType: defaultGKEProjectResourceType, + BoskosResourceType: []string{defaultGKEProjectResourceType}, BoskosAcquireTimeoutSeconds: defaultBoskosAcquireTimeoutSeconds, BoskosHeartbeatIntervalSeconds: defaultBoskosHeartbeatIntervalSeconds, - BoskosProjectsRequested: 1, + BoskosProjectsRequested: []int{1}, }, NetworkOptions: &options.NetworkOptions{ Network: "default", @@ -195,13 +210,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { localLogsDir: filepath.Join(opts.RunDir(), "logs"), } - // register flags - fs := bindFlags(d) - - // register flags for klog - klog.InitFlags(nil) - fs.AddGoFlagSet(flag.CommandLine) - return d, fs + return d } func (d *Deployer) VerifyLocationFlags() error { diff --git a/kubetest2-gke/deployer/down.go b/kubetest2-gke/deployer/down.go index c4d65e87..cc7c3ef6 100644 --- a/kubetest2-gke/deployer/down.go +++ b/kubetest2-gke/deployer/down.go @@ -30,29 +30,23 @@ func (d *Deployer) Down() error { return err } - if len(d.Projects) > 0 { - if err := d.PrepareGcpIfNeeded(d.Projects[0]); err != nil { - return err - } - - d.DeleteClusters(d.retryCount) + d.DeleteClusters(d.retryCount) - numDeletedFWRules, errCleanFirewalls := d.CleanupNetworkFirewalls(d.Projects[0], d.Network) - if errCleanFirewalls != nil { - klog.Errorf("Error cleaning-up firewall rules: %v", errCleanFirewalls) - } else { - klog.V(1).Infof("Deleted %d network firewall rules", numDeletedFWRules) - } + numDeletedFWRules, errCleanFirewalls := d.CleanupNetworkFirewalls(d.Projects[0], d.Network) + if errCleanFirewalls != nil { + klog.Errorf("Error cleaning-up firewall rules: %v", errCleanFirewalls) + } else { + klog.V(1).Infof("Deleted %d network firewall rules", numDeletedFWRules) + } - if err := d.TeardownNetwork(); err != nil { - return err - } - if err := d.DeleteSubnets(d.retryCount); err != nil { - return err - } - if err := d.DeleteNetwork(); err != nil { - return err - } + if err := d.TeardownNetwork(); err != nil { + return err + } + if err := d.DeleteSubnets(d.retryCount); err != nil { + return err + } + if err := d.DeleteNetwork(); err != nil { + return err } return nil diff --git a/kubetest2-gke/deployer/network.go b/kubetest2-gke/deployer/network.go index 2d0d16e7..fcc00676 100644 --- a/kubetest2-gke/deployer/network.go +++ b/kubetest2-gke/deployer/network.go @@ -57,7 +57,7 @@ func (d *Deployer) VerifyNetworkFlags() error { numProjects := len(d.Projects) if numProjects == 0 { - numProjects = d.BoskosProjectsRequested + numProjects = d.totalBoskosProjectsRequested } // Verify for multi-project profile. diff --git a/kubetest2-gke/deployer/options/project.go b/kubetest2-gke/deployer/options/project.go index 004aef1e..ecf1ff62 100644 --- a/kubetest2-gke/deployer/options/project.go +++ b/kubetest2-gke/deployer/options/project.go @@ -19,9 +19,9 @@ package options type ProjectOptions struct { Projects []string `flag:"~project" desc:"Comma separated list of GCP Project(s) to use for creating the cluster."` - BoskosLocation string `flag:"~boskos-location" desc:"If set, manually specifies the location of the Boskos server."` - BoskosResourceType string `flag:"~boskos-resource-type" desc:"If set, manually specifies the resource type of GCP projects to acquire from Boskos."` - BoskosAcquireTimeoutSeconds int `flag:"~boskos-acquire-timeout-seconds" desc:"How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring."` - BoskosHeartbeatIntervalSeconds int `flag:"~boskos-heartbeat-interval-seconds" desc:"How often (in seconds) to send a heartbeat to Boskos to hold the acquired resource. 0 means no heartbeat."` - BoskosProjectsRequested int `flag:"~projects-requested" desc:"Number of projects to request from Boskos. It is only respected if projects is empty, and must be larger than zero."` + BoskosLocation string `flag:"~boskos-location" desc:"If set, manually specifies the location of the Boskos server."` + BoskosAcquireTimeoutSeconds int `flag:"~boskos-acquire-timeout-seconds" desc:"How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring."` + BoskosHeartbeatIntervalSeconds int `flag:"~boskos-heartbeat-interval-seconds" desc:"How often (in seconds) to send a heartbeat to Boskos to hold the acquired resource. 0 means no heartbeat."` + BoskosResourceType []string `flag:"~boskos-resource-type" desc:"If set, manually specifies the resource type(s) of GCP projects to acquire from Boskos."` + BoskosProjectsRequested []int `flag:"~projects-requested" desc:"Number of projects to request from Boskos. It is only respected if projects is empty, and must be larger than zero."` } diff --git a/kubetest2-gke/deployer/up.go b/kubetest2-gke/deployer/up.go index fa18022f..5e81a1f0 100644 --- a/kubetest2-gke/deployer/up.go +++ b/kubetest2-gke/deployer/up.go @@ -57,10 +57,6 @@ func (d *Deployer) Up() error { } }() - // Only run prepare once for the first GCP project. - if err := d.PrepareGcpIfNeeded(d.Projects[0]); err != nil { - return err - } if err := d.CreateNetwork(); err != nil { return err } @@ -278,10 +274,6 @@ func (d *Deployer) TestSetup() error { return nil } - // Only run prepare once for the first GCP project. - if err := d.PrepareGcpIfNeeded(d.Projects[0]); err != nil { - return err - } if _, err := d.Kubeconfig(); err != nil { return err } @@ -328,18 +320,26 @@ func (d *Deployer) Kubeconfig() (string, error) { // verifyCommonFlags validates flags for up phase. func (d *Deployer) VerifyUpFlags() error { - if len(d.Projects) == 0 && d.BoskosProjectsRequested <= 0 { - return fmt.Errorf("either --project or --projects-requested with a value larger than 0 must be set for GKE deployment") + if len(d.Projects) == 0 { + for _, num := range d.BoskosProjectsRequested { + d.totalBoskosProjectsRequested += num + } + if d.totalBoskosProjectsRequested <= 0 { + return fmt.Errorf("either --project or --projects-requested with a value larger than 0 must be set for GKE deployment") + } + if len(d.BoskosProjectsRequested) != len(d.BoskosResourceType) { + return fmt.Errorf("the length of --project-requested and --boskos-resource-type must be the same") + } } if len(d.Clusters) == 0 { - if len(d.Projects) > 1 || d.BoskosProjectsRequested > 1 { + if len(d.Projects) > 1 || d.totalBoskosProjectsRequested > 1 { return fmt.Errorf("explicit --cluster-name must be set for multi-project profile") } if err := d.ClusterOptions.Validate(); err != nil { return err } - d.Clusters = generateClusterNames(d.ClusterOptions.NumClusters, d.kubetest2CommonOptions.RunID()) + d.Clusters = generateClusterNames(d.ClusterOptions.NumClusters, d.Kubetest2CommonOptions.RunID()) } else { klog.V(0).Infof("explicit --cluster-name specified, ignoring --num-clusters") } From fda5db5496eb8c36e09caec2068d342536908862 Mon Sep 17 00:00:00 2001 From: Chi Zhang Date: Sun, 18 Jul 2021 17:57:20 -0700 Subject: [PATCH 3/4] Allow kubetest2 gke --down to run individually --- kubetest2-gke/deployer/common.go | 37 ++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/kubetest2-gke/deployer/common.go b/kubetest2-gke/deployer/common.go index cce582d9..f1670a5e 100644 --- a/kubetest2-gke/deployer/common.go +++ b/kubetest2-gke/deployer/common.go @@ -92,22 +92,6 @@ func (d *Deployer) Initialize() error { } } } - - // Multi-cluster name adjustment - numProjects := len(d.Projects) - d.projectClustersLayout = make(map[string][]cluster, numProjects) - if numProjects > 1 { - if err := buildProjectClustersLayout(d.Projects, d.Clusters, d.projectClustersLayout); err != nil { - return fmt.Errorf("failed to build the project clusters layout: %v", err) - } - } else { - // Backwards compatible construction - clusters := make([]cluster, len(d.Clusters)) - for i, clusterName := range d.Clusters { - clusters[i] = cluster{i, clusterName} - } - d.projectClustersLayout[d.Projects[0]] = clusters - } } if d.Kubetest2CommonOptions.ShouldDown() { @@ -116,6 +100,27 @@ func (d *Deployer) Initialize() error { } } + // Multi-cluster name adjustment + numProjects := len(d.Projects) + d.projectClustersLayout = make(map[string][]cluster, numProjects) + if numProjects > 1 { + if err := buildProjectClustersLayout(d.Projects, d.Clusters, d.projectClustersLayout); err != nil { + return fmt.Errorf("failed to build the project clusters layout: %v", err) + } + } else { + // Backwards compatible construction + clusters := make([]cluster, len(d.Clusters)) + for i, clusterName := range d.Clusters { + clusters[i] = cluster{i, clusterName} + } + d.projectClustersLayout[d.Projects[0]] = clusters + } + + // Prepare the GCP environment for the following operations. + if err := d.PrepareGcpIfNeeded(d.Projects[0]); err != nil { + return err + } + return nil } From d6a992173c7fa3f3b3259e08d2f906f230698b35 Mon Sep 17 00:00:00 2001 From: Chi Zhang Date: Sun, 18 Jul 2021 17:58:02 -0700 Subject: [PATCH 4/4] Add a new test case for version.go --- kubetest2-gke/deployer/version_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kubetest2-gke/deployer/version_test.go b/kubetest2-gke/deployer/version_test.go index 09391085..01fdc556 100644 --- a/kubetest2-gke/deployer/version_test.go +++ b/kubetest2-gke/deployer/version_test.go @@ -161,6 +161,12 @@ func TestIsClusterVersionMatch(t *testing.T) { target: "1.19.10-gke.1000", expectMatched: false, }, + { + desc: "version having more parts than the target cannot be matched", + version: "1.19.10.1.2", + target: "1.19.10-gke.1000", + expectMatched: false, + }, } for _, tc := range testCases {