diff --git a/tests/e2e/kubetest2-kops/deployer/boskos.go b/tests/e2e/kubetest2-kops/deployer/boskos.go new file mode 100644 index 0000000000000..abcc155e70ae5 --- /dev/null +++ b/tests/e2e/kubetest2-kops/deployer/boskos.go @@ -0,0 +1,123 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployer + +import ( + "encoding/json" + "fmt" + "os" + + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/boskos" +) + +func (d *deployer) acquireBoskosAWSAccount() error { + klog.V(1).Info("Acquiring AWS account from Boskos") + + boskosClient, err := boskos.NewClient(d.BoskosLocation) + if err != nil { + return fmt.Errorf("failed to make boskos client: %s", err) + } + d.boskos = boskosClient + + resource, err := boskos.Acquire( + d.boskos, + d.BoskosAWSResourceType, + d.BoskosAcquireTimeout, + d.BoskosHeartbeatInterval, + d.boskosHeartbeatClose, + ) + if err != nil { + return fmt.Errorf("init failed to get aws account from boskos: %s", err) + } + d.boskosAWSAccount = resource.Name + klog.V(1).Infof("Got aws account %s from boskos", d.boskosAWSAccount) + + if resource.UserData == nil { + return fmt.Errorf("boskos resource %s has nil UserData", resource.Name) + } + + jsonBytes, err := json.Marshal(resource.UserData) + if err != nil { + // Use %w to wrap the error for better debugging + return fmt.Errorf("failed to marshal boskos user data: %w", err) + } + + var data struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + SessionToken string `json:"session_token"` + } + + // 2. Unmarshal the JSON bytes (from the map) into the 'data' struct + if err := json.Unmarshal(jsonBytes, &data); err != nil { + return fmt.Errorf("failed to unmarshal boskos user data: %w", err) + } + + os.Setenv("AWS_ACCESS_KEY_ID", data.AccessKeyID) + os.Setenv("AWS_SECRET_ACCESS_KEY", data.SecretAccessKey) + os.Setenv("AWS_SESSION_TOKEN", data.SessionToken) + + return nil +} + +func (d *deployer) acquireBoskosGCPProject() error { + klog.V(1).Info("No GCP project provided, acquiring from Boskos") + + boskosClient, err := boskos.NewClient(d.BoskosLocation) + if err != nil { + return fmt.Errorf("failed to make boskos client: %s", err) + } + d.boskos = boskosClient + + resource, err := boskos.Acquire( + d.boskos, + d.BoskosGCPResourceType, + d.BoskosAcquireTimeout, + d.BoskosHeartbeatInterval, + d.boskosHeartbeatClose, + ) + if err != nil { + return fmt.Errorf("init failed to get project from boskos: %s", err) + } + d.GCPProject = resource.Name + klog.V(1).Infof("Got project %s from boskos", d.GCPProject) + return nil +} + +func (d *deployer) releaseBoskosResources() error { + if d.boskos == nil { + return nil + } + klog.V(2).Info("releasing boskos project") + resources := []string{} + if d.GCPProject != "" { + resources = append(resources, d.GCPProject) + } + if d.boskosAWSAccount != "" { + resources = append(resources, d.boskosAWSAccount) + } + err := boskos.Release( + d.boskos, + resources, + d.boskosHeartbeatClose, + ) + if err != nil { + return fmt.Errorf("down failed to release boskos project: %s", err) + } + return nil +} diff --git a/tests/e2e/kubetest2-kops/deployer/common.go b/tests/e2e/kubetest2-kops/deployer/common.go index af374ce7ff2d6..3174a2fd5b37c 100644 --- a/tests/e2e/kubetest2-kops/deployer/common.go +++ b/tests/e2e/kubetest2-kops/deployer/common.go @@ -29,7 +29,6 @@ import ( "k8s.io/kops/tests/e2e/kubetest2-kops/gce" "k8s.io/kops/tests/e2e/pkg/target" "k8s.io/kops/tests/e2e/pkg/util" - "sigs.k8s.io/kubetest2/pkg/boskos" ) func (d *deployer) init() error { @@ -53,6 +52,11 @@ func (d *deployer) initialize() error { switch d.CloudProvider { case "aws": + if d.UseBoskosAWS { + if err := d.acquireBoskosAWSAccount(); err != nil { + return err + } + } client, err := aws.NewClient(context.Background()) if err != nil { return fmt.Errorf("init failed to build AWS client: %w", err) @@ -91,26 +95,9 @@ func (d *deployer) initialize() error { d.SSHUser = "root" case "gce": if d.GCPProject == "" { - klog.V(1).Info("No GCP project provided, acquiring from Boskos") - - boskosClient, err := boskos.NewClient(d.BoskosLocation) - if err != nil { - return fmt.Errorf("failed to make boskos client: %s", err) - } - d.boskos = boskosClient - - resource, err := boskos.Acquire( - d.boskos, - d.BoskosResourceType, - d.BoskosAcquireTimeout, - d.BoskosHeartbeatInterval, - d.boskosHeartbeatClose, - ) - if err != nil { - return fmt.Errorf("init failed to get project from boskos: %s", err) + if err := d.acquireBoskosGCPProject(); err != nil { + return err } - d.GCPProject = resource.Name - klog.V(1).Infof("Got project %s from boskos", d.GCPProject) if d.SSHPrivateKeyPath == "" { d.SSHPrivateKeyPath = os.Getenv("GCE_SSH_PRIVATE_KEY_FILE") diff --git a/tests/e2e/kubetest2-kops/deployer/deployer.go b/tests/e2e/kubetest2-kops/deployer/deployer.go index 7b79e24176890..bc5fc93a1bdfa 100644 --- a/tests/e2e/kubetest2-kops/deployer/deployer.go +++ b/tests/e2e/kubetest2-kops/deployer/deployer.go @@ -105,10 +105,14 @@ type deployer struct { // boskos struct field will be non-nil when the deployer is // using boskos to acquire a GCP project boskos *client.Client + boskosAWSAccount string BoskosLocation string `flag:"boskos-location" desc:"If set, manually specifies the location of the Boskos server."` BoskosAcquireTimeout time.Duration `flag:"boskos-acquire-timeout" desc:"How long should boskos wait to acquire a resource before timing out"` BoskosHeartbeatInterval time.Duration `flag:"boskos-heartbeat-interval" desc:"How often should boskos send a heartbeat to Boskos to hold the acquired resource. 0 means no heartbeat."` - BoskosResourceType string `flag:"boskos-resource-type" desc:"If set, manually specifies the resource type of GCP projects to acquire from Boskos."` + BoskosGCPResourceType string `flag:"boskos-gcp-resource-type" desc:"If set, manually specifies the resource type of GCP projects to acquire from Boskos."` + BoskosAWSResourceType string `flag:"boskos-aws-resource-type" desc:"If set, manually specifies the resource type of AWS accounts to acquire from Boskos."` + UseBoskosAWS bool `flag:"use-boskos-aws" desc:"If set, use boskos to acquire AWS accounts."` + // this channel serves as a signal channel for the hearbeat goroutine // so that it can be explicitly closed boskosHeartbeatClose chan struct{} @@ -139,7 +143,8 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { ValidationCount: 10, ValidationInterval: 10 * time.Second, BoskosLocation: "http://boskos.test-pods.svc.cluster.local.", - BoskosResourceType: "gce-project", + BoskosGCPResourceType: "gce-project", + BoskosAWSResourceType: "aws-account", BoskosAcquireTimeout: 5 * time.Minute, BoskosHeartbeatInterval: 5 * time.Minute, Tags: []string{ diff --git a/tests/e2e/kubetest2-kops/deployer/down.go b/tests/e2e/kubetest2-kops/deployer/down.go index 2904a1db9e5e2..04b1240d5881e 100644 --- a/tests/e2e/kubetest2-kops/deployer/down.go +++ b/tests/e2e/kubetest2-kops/deployer/down.go @@ -18,13 +18,11 @@ package deployer import ( "context" - "fmt" "strings" "k8s.io/klog/v2" "k8s.io/kops/tests/e2e/kubetest2-kops/gce" "k8s.io/kops/tests/e2e/pkg/kops" - "sigs.k8s.io/kubetest2/pkg/boskos" "sigs.k8s.io/kubetest2/pkg/exec" ) @@ -86,16 +84,5 @@ func (d *deployer) Down() error { } } - if d.boskos != nil { - klog.V(2).Info("releasing boskos project") - err := boskos.Release( - d.boskos, - []string{d.GCPProject}, - d.boskosHeartbeatClose, - ) - if err != nil { - return fmt.Errorf("down failed to release boskos project: %s", err) - } - } - return nil + return d.releaseBoskosResources() }