diff --git a/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go b/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go index 6b7d1633309f..d2e623826f97 100644 --- a/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go +++ b/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go @@ -17,15 +17,18 @@ limitations under the License. package aws import ( + "context" "fmt" "reflect" "strings" "sync" "time" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/autoscaling" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "k8s.io/autoscaler/cluster-autoscaler/config/dynamic" klog "k8s.io/klog/v2" ) @@ -36,12 +39,20 @@ const ( placeholderUnfulfillableStatus = "placeholder-cannot-be-fulfilled" ) +// getStringValue returns the value of a string pointer or empty string if nil +func getStringValue(s *string) string { + if s == nil { + return "" + } + return *s +} + type asgCache struct { registeredAsgs map[AwsRef]*asg asgToInstances map[AwsRef][]AwsInstanceRef instanceToAsg map[AwsInstanceRef]*asg instanceStatus map[AwsInstanceRef]*string - instanceLifecycle map[AwsInstanceRef]*string + instanceLifecycle map[AwsInstanceRef]autoscalingtypes.LifecycleState asgInstanceTypeCache *instanceTypeExpirationStore mutex sync.Mutex awsService *awsWrapper @@ -60,8 +71,8 @@ type launchTemplate struct { type mixedInstancesPolicy struct { launchTemplate *launchTemplate instanceTypesOverrides []string - instanceRequirementsOverrides *autoscaling.InstanceRequirements - instanceRequirements *ec2.InstanceRequirements + instanceRequirementsOverrides *autoscalingtypes.InstanceRequirements + instanceRequirements *ec2types.InstanceRequirements } type asg struct { @@ -76,7 +87,7 @@ type asg struct { LaunchConfigurationName string LaunchTemplate *launchTemplate MixedInstancesPolicy *mixedInstancesPolicy - Tags []*autoscaling.TagDescription + Tags []autoscalingtypes.TagDescription } func newASGCache(awsService *awsWrapper, explicitSpecs []string, autoDiscoverySpecs []asgAutoDiscoveryConfig) (*asgCache, error) { @@ -86,7 +97,7 @@ func newASGCache(awsService *awsWrapper, explicitSpecs []string, autoDiscoverySp asgToInstances: make(map[AwsRef][]AwsInstanceRef), instanceToAsg: make(map[AwsInstanceRef]*asg), instanceStatus: make(map[AwsInstanceRef]*string), - instanceLifecycle: make(map[AwsInstanceRef]*string), + instanceLifecycle: make(map[AwsInstanceRef]autoscalingtypes.LifecycleState), asgInstanceTypeCache: newAsgInstanceTypeCache(awsService), interrupt: make(chan struct{}), asgAutoDiscoverySpecs: autoDiscoverySpecs, @@ -243,12 +254,12 @@ func (m *asgCache) InstanceStatus(ref AwsInstanceRef) (*string, error) { return nil, fmt.Errorf("could not find instance %v", ref) } -func (m *asgCache) findInstanceLifecycle(ref AwsInstanceRef) (*string, error) { +func (m *asgCache) findInstanceLifecycle(ref AwsInstanceRef) (autoscalingtypes.LifecycleState, error) { if lifecycle, found := m.instanceLifecycle[ref]; found { return lifecycle, nil } - return nil, fmt.Errorf("could not find instance %v", ref) + return "", fmt.Errorf("could not find instance %v", ref) } func (m *asgCache) SetAsgSize(asg *asg, size int) error { @@ -259,14 +270,15 @@ func (m *asgCache) SetAsgSize(asg *asg, size int) error { } func (m *asgCache) setAsgSizeNoLock(asg *asg, size int) error { + ctx := context.Background() params := &autoscaling.SetDesiredCapacityInput{ AutoScalingGroupName: aws.String(asg.Name), - DesiredCapacity: aws.Int64(int64(size)), + DesiredCapacity: aws.Int32(int32(size)), HonorCooldown: aws.Bool(false), } klog.V(0).Infof("Setting asg %s size to %d", asg.Name, size) start := time.Now() - _, err := m.awsService.SetDesiredCapacity(params) + _, err := m.awsService.SetDesiredCapacity(ctx, params) observeAWSRequest("SetDesiredCapacity", err, start) if err != nil { return err @@ -356,21 +368,21 @@ func (m *asgCache) DeleteInstances(instances []*AwsInstanceRef) error { return err } - if lifecycle != nil && - *lifecycle == autoscaling.LifecycleStateTerminated || - *lifecycle == autoscaling.LifecycleStateTerminating || - *lifecycle == autoscaling.LifecycleStateTerminatingWait || - *lifecycle == autoscaling.LifecycleStateTerminatingProceed { - klog.V(2).Infof("instance %s is already terminating in state %s, will skip instead", instance.Name, *lifecycle) + if lifecycle == autoscalingtypes.LifecycleStateTerminated || + lifecycle == autoscalingtypes.LifecycleStateTerminating || + lifecycle == autoscalingtypes.LifecycleStateTerminatingWait || + lifecycle == autoscalingtypes.LifecycleStateTerminatingProceed { + klog.V(2).Infof("instance %s is already terminating in state %s, will skip instead", instance.Name, lifecycle) continue } + ctx := context.Background() params := &autoscaling.TerminateInstanceInAutoScalingGroupInput{ InstanceId: aws.String(instance.Name), ShouldDecrementDesiredCapacity: aws.Bool(true), } start := time.Now() - resp, err := m.awsService.TerminateInstanceInAutoScalingGroup(params) + resp, err := m.awsService.TerminateInstanceInAutoScalingGroup(ctx, params) observeAWSRequest("TerminateInstanceInAutoScalingGroup", err, start) if err != nil { return err @@ -421,7 +433,7 @@ func (m *asgCache) regenerate() error { newInstanceToAsgCache := make(map[AwsInstanceRef]*asg) newAsgToInstancesCache := make(map[AwsRef][]AwsInstanceRef) newInstanceStatusMap := make(map[AwsInstanceRef]*string) - newInstanceLifecycleMap := make(map[AwsInstanceRef]*string) + newInstanceLifecycleMap := make(map[AwsInstanceRef]autoscalingtypes.LifecycleState) // Fetch details of all ASGs refreshNames := m.buildAsgNames() @@ -447,7 +459,8 @@ func (m *asgCache) regenerate() error { // Register or update ASGs exists := make(map[AwsRef]bool) - for _, group := range groups { + for i := range groups { + group := &groups[i] asg, err := m.buildAsgFromAWS(group) if err != nil { return err @@ -458,10 +471,11 @@ func (m *asgCache) regenerate() error { newAsgToInstancesCache[asg.AwsRef] = make([]AwsInstanceRef, len(group.Instances)) - for i, instance := range group.Instances { + for j := range group.Instances { + instance := &group.Instances[j] ref := m.buildInstanceRefFromAWS(instance) newInstanceToAsgCache[ref] = asg - newAsgToInstancesCache[asg.AwsRef][i] = ref + newAsgToInstancesCache[asg.AwsRef][j] = ref newInstanceStatusMap[ref] = instance.HealthStatus newInstanceLifecycleMap[ref] = instance.LifecycleState } @@ -497,9 +511,10 @@ func (m *asgCache) regenerate() error { return nil } -func (m *asgCache) createPlaceholdersForDesiredNonStartedInstances(groups []*autoscaling.Group) []*autoscaling.Group { - for _, g := range groups { - desired := *g.DesiredCapacity +func (m *asgCache) createPlaceholdersForDesiredNonStartedInstances(groups []autoscalingtypes.AutoScalingGroup) []autoscalingtypes.AutoScalingGroup { + for idx := range groups { + g := &groups[idx] + desired := int64(*g.DesiredCapacity) realInstances := int64(len(g.Instances)) if desired <= realInstances { continue @@ -519,9 +534,10 @@ func (m *asgCache) createPlaceholdersForDesiredNonStartedInstances(groups []*aut for i := realInstances; i < desired; i++ { id := fmt.Sprintf("%s-%s-%d", placeholderInstanceNamePrefix, *g.AutoScalingGroupName, i) - g.Instances = append(g.Instances, &autoscaling.Instance{ + az := g.AvailabilityZones[0] + g.Instances = append(g.Instances, autoscalingtypes.Instance{ InstanceId: &id, - AvailabilityZone: g.AvailabilityZones[0], + AvailabilityZone: &az, HealthStatus: &healthStatus, }) } @@ -529,13 +545,14 @@ func (m *asgCache) createPlaceholdersForDesiredNonStartedInstances(groups []*aut return groups } -func (m *asgCache) isNodeGroupAvailable(group *autoscaling.Group) (bool, error) { +func (m *asgCache) isNodeGroupAvailable(group *autoscalingtypes.AutoScalingGroup) (bool, error) { + ctx := context.Background() input := &autoscaling.DescribeScalingActivitiesInput{ AutoScalingGroupName: group.AutoScalingGroupName, } start := time.Now() - response, err := m.awsService.DescribeScalingActivities(input) + response, err := m.awsService.DescribeScalingActivities(ctx, input) observeAWSRequest("DescribeScalingActivities", err, start) if err != nil { return true, err // If we can't describe the scaling activities we assume the node group is available @@ -547,8 +564,8 @@ func (m *asgCache) isNodeGroupAvailable(group *autoscaling.Group) (bool, error) lut := a.lastUpdateTime if activity.StartTime.Before(lut) { break - } else if *activity.StatusCode == "Failed" { - klog.Warningf("ASG %s scaling failed with %s", asgRef.Name, *activity) + } else if activity.StatusCode == autoscalingtypes.ScalingActivityStatusCodeFailed { + klog.Warningf("ASG %s scaling failed with %v", asgRef.Name, activity) return false, nil } } else { @@ -558,11 +575,11 @@ func (m *asgCache) isNodeGroupAvailable(group *autoscaling.Group) (bool, error) return true, nil } -func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { +func (m *asgCache) buildAsgFromAWS(g *autoscalingtypes.AutoScalingGroup) (*asg, error) { spec := dynamic.NodeGroupSpec{ - Name: aws.StringValue(g.AutoScalingGroupName), - MinSize: int(aws.Int64Value(g.MinSize)), - MaxSize: int(aws.Int64Value(g.MaxSize)), + Name: *g.AutoScalingGroupName, + MinSize: int(*g.MinSize), + MaxSize: int(*g.MaxSize), SupportScaleToZero: scaleToZeroSupported, } @@ -575,9 +592,9 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { minSize: spec.MinSize, maxSize: spec.MaxSize, - curSize: int(aws.Int64Value(g.DesiredCapacity)), - AvailabilityZones: aws.StringValueSlice(g.AvailabilityZones), - LaunchConfigurationName: aws.StringValue(g.LaunchConfigurationName), + curSize: int(*g.DesiredCapacity), + AvailabilityZones: g.AvailabilityZones, + LaunchConfigurationName: getStringValue(g.LaunchConfigurationName), Tags: g.Tags, } @@ -586,7 +603,7 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { } if g.MixedInstancesPolicy != nil { - getInstanceTypes := func(overrides []*autoscaling.LaunchTemplateOverrides) []string { + getInstanceTypes := func(overrides []autoscalingtypes.LaunchTemplateOverrides) []string { res := []string{} for _, override := range overrides { if override.InstanceType != nil { @@ -596,7 +613,7 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { return res } - getInstanceTypeRequirements := func(overrides []*autoscaling.LaunchTemplateOverrides) *autoscaling.InstanceRequirements { + getInstanceTypeRequirements := func(overrides []autoscalingtypes.LaunchTemplateOverrides) *autoscalingtypes.InstanceRequirements { if len(overrides) == 1 && overrides[0].InstanceRequirements != nil { return overrides[0].InstanceRequirements } @@ -625,8 +642,8 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { return asg, nil } -func (m *asgCache) getInstanceRequirementsFromMixedInstancesPolicy(policy *mixedInstancesPolicy) (*ec2.InstanceRequirements, error) { - instanceRequirements := &ec2.InstanceRequirements{} +func (m *asgCache) getInstanceRequirementsFromMixedInstancesPolicy(policy *mixedInstancesPolicy) (*ec2types.InstanceRequirements, error) { + instanceRequirements := &ec2types.InstanceRequirements{} if policy.instanceRequirementsOverrides != nil { var err error instanceRequirements, err = m.awsService.getEC2RequirementsFromAutoscaling(policy.instanceRequirementsOverrides) @@ -646,11 +663,11 @@ func (m *asgCache) getInstanceRequirementsFromMixedInstancesPolicy(policy *mixed return instanceRequirements, nil } -func (m *asgCache) buildInstanceRefFromAWS(instance *autoscaling.Instance) AwsInstanceRef { - providerID := fmt.Sprintf("aws:///%s/%s", aws.StringValue(instance.AvailabilityZone), aws.StringValue(instance.InstanceId)) +func (m *asgCache) buildInstanceRefFromAWS(instance *autoscalingtypes.Instance) AwsInstanceRef { + providerID := fmt.Sprintf("aws:///%s/%s", getStringValue(instance.AvailabilityZone), getStringValue(instance.InstanceId)) return AwsInstanceRef{ ProviderID: providerID, - Name: aws.StringValue(instance.InstanceId), + Name: getStringValue(instance.InstanceId), } } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go index 3117b1c73654..9344b39ec534 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go @@ -436,7 +436,7 @@ func BuildAWS(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscover if opts.AWSUseStaticInstanceList { klog.Warningf("Using static EC2 Instance Types, this list could be outdated. Last update time: %s", lastUpdateTime) } else { - generatedInstanceTypes, err := GenerateEC2InstanceTypes(sdkProvider.session) + generatedInstanceTypes, err := GenerateEC2InstanceTypes(sdkProvider.cfg) if err != nil { klog.Errorf("Failed to generate AWS EC2 Instance Types: %v, falling back to static list with last update time: %s", err, lastUpdateTime) } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_manager.go b/cluster-autoscaler/cloudprovider/aws/aws_manager.go index 6a649a5860a3..d89f0fe1c884 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_manager.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_manager.go @@ -27,16 +27,17 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/eks" apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/autoscaling" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/eks" "k8s.io/autoscaler/cluster-autoscaler/config" "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" ) @@ -66,7 +67,7 @@ type asgTemplate struct { InstanceType *InstanceType Region string Zone string - Tags []*autoscaling.TagDescription + Tags []autoscalingtypes.TagDescription } // createAwsManagerInternal allows for custom objects to be passed in by tests @@ -76,11 +77,15 @@ func createAWSManagerInternal( awsService *awsWrapper, instanceTypes map[string]*InstanceType, ) (*AwsManager, error) { - klog.Infof("AWS SDK Version: %s", aws.SDKVersion) + klog.Infof("AWS SDK Version: v2") if awsService == nil { - sess := awsSDKProvider.session - awsService = &awsWrapper{autoscaling.New(sess), ec2.New(sess), eks.New(sess)} + cfg := awsSDKProvider.cfg + awsService = &awsWrapper{ + autoscaling.NewFromConfig(cfg), + ec2.NewFromConfig(cfg), + eks.NewFromConfig(cfg), + } } specs, err := parseASGAutoDiscoverySpecs(discoveryOpts) @@ -363,18 +368,18 @@ func (m *AwsManager) updateCapacityWithRequirementsOverrides(capacity *apiv1.Res instanceRequirements := policy.instanceRequirements if instanceRequirements.VCpuCount != nil && instanceRequirements.VCpuCount.Min != nil { - (*capacity)[apiv1.ResourceCPU] = *resource.NewQuantity(*instanceRequirements.VCpuCount.Min, resource.DecimalSI) + (*capacity)[apiv1.ResourceCPU] = *resource.NewQuantity(int64(*instanceRequirements.VCpuCount.Min), resource.DecimalSI) } if instanceRequirements.MemoryMiB != nil && instanceRequirements.MemoryMiB.Min != nil { - (*capacity)[apiv1.ResourceMemory] = *resource.NewQuantity(*instanceRequirements.MemoryMiB.Min*1024*1024, resource.DecimalSI) + (*capacity)[apiv1.ResourceMemory] = *resource.NewQuantity(int64(*instanceRequirements.MemoryMiB.Min)*1024*1024, resource.DecimalSI) } for _, manufacturer := range instanceRequirements.AcceleratorManufacturers { - if *manufacturer == autoscaling.AcceleratorManufacturerNvidia { + if manufacturer == ec2types.AcceleratorManufacturerNvidia { for _, acceleratorType := range instanceRequirements.AcceleratorTypes { - if *acceleratorType == autoscaling.AcceleratorTypeGpu { - (*capacity)[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(*instanceRequirements.AcceleratorCount.Min, resource.DecimalSI) + if acceleratorType == ec2types.AcceleratorTypeGpu { + (*capacity)[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(int64(*instanceRequirements.AcceleratorCount.Min), resource.DecimalSI) } } } @@ -396,7 +401,7 @@ func buildGenericLabels(template *asgTemplate, nodeName string) map[string]strin return result } -func extractLabelsFromAsg(tags []*autoscaling.TagDescription) map[string]string { +func extractLabelsFromAsg(tags []autoscalingtypes.TagDescription) map[string]string { result := make(map[string]string) for _, tag := range tags { @@ -418,22 +423,23 @@ func extractLabelsFromAsg(tags []*autoscaling.TagDescription) map[string]string return result } -func extractAutoscalingOptionsFromTags(tags []*autoscaling.TagDescription) map[string]string { +func extractAutoscalingOptionsFromTags(tags []autoscalingtypes.TagDescription) map[string]string { options := make(map[string]string) for _, tag := range tags { - if !strings.HasPrefix(aws.StringValue(tag.Key), optionsTagsPrefix) { + key := getStringValue(tag.Key) + if !strings.HasPrefix(key, optionsTagsPrefix) { continue } - splits := strings.Split(aws.StringValue(tag.Key), optionsTagsPrefix) + splits := strings.Split(key, optionsTagsPrefix) if len(splits) != 2 || splits[1] == "" { continue } - options[splits[1]] = aws.StringValue(tag.Value) + options[splits[1]] = getStringValue(tag.Value) } return options } -func extractAllocatableResourcesFromAsg(tags []*autoscaling.TagDescription) map[string]*resource.Quantity { +func extractAllocatableResourcesFromAsg(tags []autoscalingtypes.TagDescription) map[string]*resource.Quantity { result := make(map[string]*resource.Quantity) for _, tag := range tags { @@ -476,7 +482,7 @@ func extractAllocatableResourcesFromTags(tags map[string]string) map[string]*res return result } -func extractTaintsFromAsg(tags []*autoscaling.TagDescription) []apiv1.Taint { +func extractTaintsFromAsg(tags []autoscalingtypes.TagDescription) []apiv1.Taint { taints := make([]apiv1.Taint, 0) for _, tag := range tags { diff --git a/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider.go b/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider.go index b1f33442f71b..5c2c7b1de1cf 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider.go @@ -17,18 +17,17 @@ limitations under the License. package aws import ( + "context" "fmt" "io" "os" "strconv" "strings" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" "gopkg.in/gcfg.v1" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/ec2metadata" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/endpoints" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/request" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/session" "k8s.io/autoscaler/cluster-autoscaler/version" provider_aws "k8s.io/cloud-provider-aws/pkg/providers/v1" "k8s.io/klog/v2" @@ -42,59 +41,60 @@ import ( // // t.Setenv("AWS_REGION", "fanghorn") func createAWSSDKProvider(configReader io.Reader) (*awsSDKProvider, error) { - cfg, err := readAWSCloudConfig(configReader) + cloudCfg, err := readAWSCloudConfig(configReader) if err != nil { klog.Errorf("Couldn't read config: %v", err) return nil, err } - if err = validateOverrides(cfg); err != nil { + if err = validateOverrides(cloudCfg); err != nil { klog.Errorf("Unable to validate custom endpoint overrides: %v", err) return nil, err } - config := aws.NewConfig(). - WithRegion(getRegion()). - WithEndpointResolver(getResolver(cfg)) - - config, err = setMaxRetriesFromEnv(config) - if err != nil { - return nil, err + ctx := context.Background() + + // Load the AWS SDK v2 config + var opts []func(*config.LoadOptions) error + + // Get region + region := getRegion(ctx) + if region != "" { + opts = append(opts, config.WithRegion(region)) } - - sess, err := session.NewSession(config) + + // Set max retries from environment + maxRetries := os.Getenv("AWS_MAX_ATTEMPTS") + if maxRetries != "" { + num, err := strconv.Atoi(maxRetries) + if err != nil { + return nil, err + } + opts = append(opts, config.WithRetryMaxAttempts(num)) + } + + // Add custom endpoint resolver if configured + if len(cloudCfg.ServiceOverride) > 0 { + opts = append(opts, config.WithEndpointResolverWithOptions(getResolverV2(cloudCfg))) + } + + // TODO: Add user agent middleware for cluster-autoscaler version tracking + // User agent handling in SDK v2 requires custom middleware implementation + + cfg, err := config.LoadDefaultConfig(ctx, opts...) if err != nil { return nil, err } - // add cluster-autoscaler to the user-agent to make it easier to identify - agent := fmt.Sprintf("cluster-autoscaler/v%s", version.ClusterAutoscalerVersion) - sess.Handlers.Build.PushBack(request.MakeAddToUserAgentFreeFormHandler(agent)) - provider := &awsSDKProvider{ - session: sess, + cfg: cfg, } return provider, nil } -// setMaxRetriesFromEnv sets aws config MaxRetries by reading AWS_MAX_ATTEMPTS -// aws sdk does not auto-set these so instead of having more config options we can reuse what the aws cli -// does and read AWS_MAX_ATTEMPTS from the env https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html -func setMaxRetriesFromEnv(config *aws.Config) (*aws.Config, error) { - maxRetries := os.Getenv("AWS_MAX_ATTEMPTS") - if maxRetries != "" { - num, err := strconv.Atoi(maxRetries) - if err != nil { - return nil, err - } - config = config.WithMaxRetries(num) - } - return config, nil -} - type awsSDKProvider struct { - session *session.Session + cfg aws.Config } // readAWSCloudConfig reads an instance of AWSCloudConfig from config reader. @@ -150,21 +150,11 @@ func validateOverrides(cfg *provider_aws.CloudConfig) error { return nil } -func getResolver(cfg *provider_aws.CloudConfig) endpoints.ResolverFunc { - defaultResolver := endpoints.DefaultResolver() - defaultResolverFn := func(service, region string, - optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) { - return defaultResolver.EndpointFor(service, region, optFns...) - } - if len(cfg.ServiceOverride) == 0 { - return defaultResolverFn - } - - return func(service, region string, - optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) { +func getResolverV2(cfg *provider_aws.CloudConfig) aws.EndpointResolverWithOptionsFunc { + return func(service, region string, options ...interface{}) (aws.Endpoint, error) { for _, override := range cfg.ServiceOverride { if override.Service == service && override.Region == region { - return endpoints.ResolvedEndpoint{ + return aws.Endpoint{ URL: override.URL, SigningRegion: override.SigningRegion, SigningMethod: override.SigningMethod, @@ -172,22 +162,26 @@ func getResolver(cfg *provider_aws.CloudConfig) endpoints.ResolverFunc { }, nil } } - return defaultResolver.EndpointFor(service, region, optFns...) + // Return unresolved to use default resolver + return aws.Endpoint{}, &aws.EndpointNotFoundError{} } } // getRegion deduces the current AWS Region. -func getRegion(cfg ...*aws.Config) string { +func getRegion(ctx context.Context) string { region, present := os.LookupEnv("AWS_REGION") if !present { - sess, err := session.NewSession() + // Try to get region from EC2 instance metadata + cfg, err := config.LoadDefaultConfig(ctx) if err != nil { - klog.Errorf("Error getting AWS session while retrieving region: %v", err) - } else { - svc := ec2metadata.New(sess, cfg...) - if r, err := svc.Region(); err == nil { - region = r - } + klog.Errorf("Error loading AWS config while retrieving region: %v", err) + return "" + } + + client := imds.NewFromConfig(cfg) + result, err := client.GetRegion(ctx, &imds.GetRegionInput{}) + if err == nil && result.Region != "" { + region = result.Region } } return region diff --git a/cluster-autoscaler/cloudprovider/aws/aws_util.go b/cluster-autoscaler/cloudprovider/aws/aws_util.go index b5b34cf07248..868fc41fbf1c 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_util.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_util.go @@ -17,14 +17,16 @@ limitations under the License. package aws import ( + "context" "errors" "fmt" "os" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/ec2metadata" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/session" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" ) var ( @@ -32,18 +34,29 @@ var ( ) // GenerateEC2InstanceTypes returns a map of ec2 resources -func GenerateEC2InstanceTypes(sess *session.Session) (map[string]*InstanceType, error) { - ec2Client := ec2.New(sess) +func GenerateEC2InstanceTypes(cfg aws.Config) (map[string]*InstanceType, error) { + ctx := context.Background() + ec2Client := ec2.NewFromConfig(cfg) input := ec2.DescribeInstanceTypesInput{} instanceTypes := make(map[string]*InstanceType) - if err := ec2Client.DescribeInstanceTypesPages(&input, func(page *ec2.DescribeInstanceTypesOutput, isLastPage bool) bool { + // Use pagination + var nextToken *string + for { + if nextToken != nil { + input.NextToken = nextToken + } + page, err := ec2Client.DescribeInstanceTypes(ctx, &input) + if err != nil { + return nil, err + } for _, rawInstanceType := range page.InstanceTypes { - instanceTypes[*rawInstanceType.InstanceType] = transformInstanceType(rawInstanceType) + instanceTypes[string(rawInstanceType.InstanceType)] = transformInstanceType(&rawInstanceType) + } + nextToken = page.NextToken + if nextToken == nil { + break } - return !isLastPage - }); err != nil { - return nil, err } if len(instanceTypes) == 0 { @@ -58,30 +71,30 @@ func GetStaticEC2InstanceTypes() (map[string]*InstanceType, string) { return InstanceTypes, StaticListLastUpdateTime } -func transformInstanceType(rawInstanceType *ec2.InstanceTypeInfo) *InstanceType { +func transformInstanceType(rawInstanceType *ec2types.InstanceTypeInfo) *InstanceType { instanceType := &InstanceType{ - InstanceType: *rawInstanceType.InstanceType, + InstanceType: string(rawInstanceType.InstanceType), } if rawInstanceType.MemoryInfo != nil && rawInstanceType.MemoryInfo.SizeInMiB != nil { instanceType.MemoryMb = *rawInstanceType.MemoryInfo.SizeInMiB } if rawInstanceType.VCpuInfo != nil && rawInstanceType.VCpuInfo.DefaultVCpus != nil { - instanceType.VCPU = *rawInstanceType.VCpuInfo.DefaultVCpus + instanceType.VCPU = int64(*rawInstanceType.VCpuInfo.DefaultVCpus) } if rawInstanceType.GpuInfo != nil && len(rawInstanceType.GpuInfo.Gpus) > 0 { instanceType.GPU = getGpuCount(rawInstanceType.GpuInfo) } if rawInstanceType.ProcessorInfo != nil && len(rawInstanceType.ProcessorInfo.SupportedArchitectures) > 0 { - instanceType.Architecture = interpretEc2SupportedArchitecure(*rawInstanceType.ProcessorInfo.SupportedArchitectures[0]) + instanceType.Architecture = interpretEc2SupportedArchitecure(string(rawInstanceType.ProcessorInfo.SupportedArchitectures[0])) } return instanceType } -func getGpuCount(gpuInfo *ec2.GpuInfo) int64 { +func getGpuCount(gpuInfo *ec2types.GpuInfo) int64 { var gpuCountSum int64 for _, gpu := range gpuInfo.Gpus { if gpu.Count != nil { - gpuCountSum += *gpu.Count + gpuCountSum += int64(*gpu.Count) } } return gpuCountSum @@ -107,13 +120,17 @@ func GetCurrentAwsRegion() (string, error) { region, present := os.LookupEnv("AWS_REGION") if !present { - c := aws.NewConfig(). - WithEndpoint(ec2MetaDataServiceUrl) - sess, err := session.NewSession() + ctx := context.Background() + cfg, err := config.LoadDefaultConfig(ctx, config.WithEC2IMDSEndpoint(ec2MetaDataServiceUrl)) + if err != nil { + return "", fmt.Errorf("failed to load config: %w", err) + } + client := imds.NewFromConfig(cfg) + result, err := client.GetRegion(ctx, &imds.GetRegionInput{}) if err != nil { - return "", fmt.Errorf("failed to create session") + return "", fmt.Errorf("failed to get region from metadata: %w", err) } - return ec2metadata.New(sess, c).Region() + return result.Region, nil } return region, nil diff --git a/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go b/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go index beba83acb32e..d468db6e461a 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go @@ -17,37 +17,41 @@ limitations under the License. package aws import ( + "context" "fmt" "strconv" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/eks" + ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types" apiv1 "k8s.io/api/core/v1" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/autoscaling" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/eks" klog "k8s.io/klog/v2" ) // autoScalingI is the interface abstracting specific API calls of the auto-scaling service provided by AWS SDK for use in CA type autoScalingI interface { - DescribeAutoScalingGroupsPages(input *autoscaling.DescribeAutoScalingGroupsInput, fn func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) error - DescribeLaunchConfigurations(*autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error) - DescribeScalingActivities(*autoscaling.DescribeScalingActivitiesInput) (*autoscaling.DescribeScalingActivitiesOutput, error) - SetDesiredCapacity(input *autoscaling.SetDesiredCapacityInput) (*autoscaling.SetDesiredCapacityOutput, error) - TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) + DescribeAutoScalingGroups(ctx context.Context, input *autoscaling.DescribeAutoScalingGroupsInput, optFns ...func(*autoscaling.Options)) (*autoscaling.DescribeAutoScalingGroupsOutput, error) + DescribeLaunchConfigurations(ctx context.Context, input *autoscaling.DescribeLaunchConfigurationsInput, optFns ...func(*autoscaling.Options)) (*autoscaling.DescribeLaunchConfigurationsOutput, error) + DescribeScalingActivities(ctx context.Context, input *autoscaling.DescribeScalingActivitiesInput, optFns ...func(*autoscaling.Options)) (*autoscaling.DescribeScalingActivitiesOutput, error) + SetDesiredCapacity(ctx context.Context, input *autoscaling.SetDesiredCapacityInput, optFns ...func(*autoscaling.Options)) (*autoscaling.SetDesiredCapacityOutput, error) + TerminateInstanceInAutoScalingGroup(ctx context.Context, input *autoscaling.TerminateInstanceInAutoScalingGroupInput, optFns ...func(*autoscaling.Options)) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) } // ec2I is the interface abstracting specific API calls of the EC2 service provided by AWS SDK for use in CA type ec2I interface { - DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) - DescribeLaunchTemplateVersions(input *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) - GetInstanceTypesFromInstanceRequirementsPages(input *ec2.GetInstanceTypesFromInstanceRequirementsInput, fn func(*ec2.GetInstanceTypesFromInstanceRequirementsOutput, bool) bool) error + DescribeImages(ctx context.Context, input *ec2.DescribeImagesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeImagesOutput, error) + DescribeLaunchTemplateVersions(ctx context.Context, input *ec2.DescribeLaunchTemplateVersionsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeLaunchTemplateVersionsOutput, error) + GetInstanceTypesFromInstanceRequirements(ctx context.Context, input *ec2.GetInstanceTypesFromInstanceRequirementsInput, optFns ...func(*ec2.Options)) (*ec2.GetInstanceTypesFromInstanceRequirementsOutput, error) } // eksI is the interface that represents a specific aspect of EKS (Elastic Kubernetes Service) which is provided by AWS SDK for use in CA type eksI interface { - DescribeNodegroup(input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) + DescribeNodegroup(ctx context.Context, input *eks.DescribeNodegroupInput, optFns ...func(*eks.Options)) (*eks.DescribeNodegroupOutput, error) } // awsWrapper provides several utility methods over the services provided by the AWS SDK @@ -58,12 +62,13 @@ type awsWrapper struct { } func (m *awsWrapper) getManagedNodegroupInfo(nodegroupName string, clusterName string) ([]apiv1.Taint, map[string]string, map[string]string, error) { + ctx := context.Background() params := &eks.DescribeNodegroupInput{ ClusterName: &clusterName, NodegroupName: &nodegroupName, } start := time.Now() - r, err := m.DescribeNodegroup(params) + r, err := m.DescribeNodegroup(ctx, params) observeAWSRequest("DescribeNodegroup", err, start) if err != nil { return nil, nil, nil, err @@ -77,15 +82,15 @@ func (m *awsWrapper) getManagedNodegroupInfo(nodegroupName string, clusterName s // Labels will include diskSize, amiType, capacityType, version if r.Nodegroup.DiskSize != nil { - labels["diskSize"] = strconv.FormatInt(*r.Nodegroup.DiskSize, 10) + labels["diskSize"] = strconv.FormatInt(int64(*r.Nodegroup.DiskSize), 10) } - if r.Nodegroup.AmiType != nil && len(*r.Nodegroup.AmiType) > 0 { - labels["amiType"] = *r.Nodegroup.AmiType + if r.Nodegroup.AmiType != "" { + labels["amiType"] = string(r.Nodegroup.AmiType) } - if r.Nodegroup.CapacityType != nil && len(*r.Nodegroup.CapacityType) > 0 { - labels["eks.amazonaws.com/capacityType"] = *r.Nodegroup.CapacityType + if r.Nodegroup.CapacityType != "" { + labels["eks.amazonaws.com/capacityType"] = string(r.Nodegroup.CapacityType) } if r.Nodegroup.Version != nil && len(*r.Nodegroup.Version) > 0 { @@ -99,26 +104,22 @@ func (m *awsWrapper) getManagedNodegroupInfo(nodegroupName string, clusterName s if r.Nodegroup.Labels != nil && len(r.Nodegroup.Labels) > 0 { labelsMap := r.Nodegroup.Labels for k, v := range labelsMap { - if v != nil { - labels[k] = *v - } + labels[k] = v } } if r.Nodegroup.Tags != nil && len(r.Nodegroup.Tags) > 0 { tagsMap := r.Nodegroup.Tags for k, v := range tagsMap { - if v != nil { - tags[k] = *v - } + tags[k] = v } } if r.Nodegroup.Taints != nil && len(r.Nodegroup.Taints) > 0 { taintList := r.Nodegroup.Taints for _, taint := range taintList { - if taint != nil && taint.Effect != nil && taint.Key != nil && taint.Value != nil { - formattedEffect, err := taintEksTranslator(taint) + if taint.Effect != "" && taint.Key != nil && taint.Value != nil { + formattedEffect, err := taintEksTranslator(&taint) if err != nil { return nil, nil, nil, err } @@ -134,7 +135,7 @@ func (m *awsWrapper) getManagedNodegroupInfo(nodegroupName string, clusterName s return taints, labels, tags, nil } -func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []*string) (map[string]string, error) { +func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []string) (map[string]string, error) { launchConfigurationsToInstanceType := map[string]string{} for i := 0; i < len(launchConfigToQuery); i += 50 { @@ -145,10 +146,11 @@ func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []*s } params := &autoscaling.DescribeLaunchConfigurationsInput{ LaunchConfigurationNames: launchConfigToQuery[i:end], - MaxRecords: aws.Int64(50), + MaxRecords: aws.Int32(50), } start := time.Now() - r, err := m.DescribeLaunchConfigurations(params) + ctx := context.Background() + r, err := m.DescribeLaunchConfigurations(ctx, params) observeAWSRequest("DescribeLaunchConfigurations", err, start) if err != nil { return nil, err @@ -160,12 +162,13 @@ func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []*s return launchConfigurationsToInstanceType, nil } -func (m *awsWrapper) getAutoscalingGroupsByNames(names []string) ([]*autoscaling.Group, error) { - asgs := make([]*autoscaling.Group, 0) +func (m *awsWrapper) getAutoscalingGroupsByNames(names []string) ([]autoscalingtypes.AutoScalingGroup, error) { + asgs := make([]autoscalingtypes.AutoScalingGroup, 0) if len(names) == 0 { return asgs, nil } + ctx := context.Background() // AWS only accepts up to 100 ASG names as input, describe them in batches for i := 0; i < len(names); i += maxAsgNamesPerDescribe { end := i + maxAsgNamesPerDescribe @@ -175,61 +178,82 @@ func (m *awsWrapper) getAutoscalingGroupsByNames(names []string) ([]*autoscaling } input := &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice(names[i:end]), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: names[i:end], + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), } start := time.Now() - err := m.DescribeAutoScalingGroupsPages(input, func(output *autoscaling.DescribeAutoScalingGroupsOutput, _ bool) bool { + + // Use pagination + var nextToken *string + for { + if nextToken != nil { + input.NextToken = nextToken + } + output, err := m.DescribeAutoScalingGroups(ctx, input) + if err != nil { + observeAWSRequest("DescribeAutoScalingGroups", err, start) + return nil, err + } asgs = append(asgs, output.AutoScalingGroups...) - // We return true while we want to be called with the next page of - // results, if any. - return true - }) - observeAWSRequest("DescribeAutoScalingGroupsPages", err, start) - if err != nil { - return nil, err + + nextToken = output.NextToken + if nextToken == nil { + break + } } + observeAWSRequest("DescribeAutoScalingGroups", nil, start) } return asgs, nil } -func (m *awsWrapper) getAutoscalingGroupsByTags(tags map[string]string) ([]*autoscaling.Group, error) { - asgs := make([]*autoscaling.Group, 0) +func (m *awsWrapper) getAutoscalingGroupsByTags(tags map[string]string) ([]autoscalingtypes.AutoScalingGroup, error) { + asgs := make([]autoscalingtypes.AutoScalingGroup, 0) if len(tags) == 0 { return asgs, nil } - filters := make([]*autoscaling.Filter, 0) + filters := make([]autoscalingtypes.Filter, 0) for key, value := range tags { if value != "" { - filters = append(filters, &autoscaling.Filter{ + filters = append(filters, autoscalingtypes.Filter{ Name: aws.String(fmt.Sprintf("tag:%s", key)), - Values: []*string{aws.String(value)}, + Values: []string{value}, }) } else { - filters = append(filters, &autoscaling.Filter{ + filters = append(filters, autoscalingtypes.Filter{ Name: aws.String("tag-key"), - Values: []*string{aws.String(key)}, + Values: []string{key}, }) } } + ctx := context.Background() input := &autoscaling.DescribeAutoScalingGroupsInput{ Filters: filters, - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), } start := time.Now() - err := m.DescribeAutoScalingGroupsPages(input, func(output *autoscaling.DescribeAutoScalingGroupsOutput, _ bool) bool { + + // Use pagination + var nextToken *string + for { + if nextToken != nil { + input.NextToken = nextToken + } + output, err := m.DescribeAutoScalingGroups(ctx, input) + if err != nil { + observeAWSRequest("DescribeAutoScalingGroups", err, start) + return nil, err + } asgs = append(asgs, output.AutoScalingGroups...) - // We return true while we want to be called with the next page of - // results, if any. - return true - }) - observeAWSRequest("DescribeAutoScalingGroupsPages", err, start) - if err != nil { - return nil, err + + nextToken = output.NextToken + if nextToken == nil { + break + } } + observeAWSRequest("DescribeAutoScalingGroups", nil, start) return asgs, nil } @@ -241,8 +265,8 @@ func (m *awsWrapper) getInstanceTypeByLaunchTemplate(launchTemplate *launchTempl } instanceType := "" - if templateData.InstanceType != nil { - instanceType = *templateData.InstanceType + if templateData.InstanceType != "" { + instanceType = string(templateData.InstanceType) } else if templateData.InstanceRequirements != nil && templateData.ImageId != nil { requirementsRequest, err := m.getRequirementsRequestFromEC2(templateData.InstanceRequirements) if err != nil { @@ -282,14 +306,15 @@ func (m *awsWrapper) getInstanceTypeFromRequirementsOverrides(policy *mixedInsta return instanceType, nil } -func (m *awsWrapper) getLaunchTemplateData(templateName string, templateVersion string) (*ec2.ResponseLaunchTemplateData, error) { +func (m *awsWrapper) getLaunchTemplateData(templateName string, templateVersion string) (*ec2types.ResponseLaunchTemplateData, error) { + ctx := context.Background() describeTemplateInput := &ec2.DescribeLaunchTemplateVersionsInput{ LaunchTemplateName: aws.String(templateName), - Versions: []*string{aws.String(templateVersion)}, + Versions: []string{templateVersion}, } start := time.Now() - describeData, err := m.DescribeLaunchTemplateVersions(describeTemplateInput) + describeData, err := m.DescribeLaunchTemplateVersions(ctx, describeTemplateInput) observeAWSRequest("DescribeLaunchTemplateVersions", err, start) if err != nil { return nil, err @@ -304,20 +329,21 @@ func (m *awsWrapper) getLaunchTemplateData(templateName string, templateVersion return describeData.LaunchTemplateVersions[0].LaunchTemplateData, nil } -func (m *awsWrapper) getInstanceTypeFromInstanceRequirements(imageId string, requirementsRequest *ec2.InstanceRequirementsRequest) (string, error) { +func (m *awsWrapper) getInstanceTypeFromInstanceRequirements(imageId string, requirementsRequest *ec2types.InstanceRequirementsRequest) (string, error) { + ctx := context.Background() describeImagesInput := &ec2.DescribeImagesInput{ - ImageIds: []*string{aws.String(imageId)}, + ImageIds: []string{imageId}, } start := time.Now() - describeImagesOutput, err := m.DescribeImages(describeImagesInput) + describeImagesOutput, err := m.DescribeImages(ctx, describeImagesInput) observeAWSRequest("DescribeImages", err, start) if err != nil { return "", err } - imageArchitectures := []*string{} - imageVirtualizationTypes := []*string{} + imageArchitectures := []ec2types.ArchitectureType{} + imageVirtualizationTypes := []ec2types.VirtualizationType{} for _, image := range describeImagesOutput.Images { imageArchitectures = append(imageArchitectures, image.Architecture) imageVirtualizationTypes = append(imageVirtualizationTypes, image.VirtualizationType) @@ -331,16 +357,27 @@ func (m *awsWrapper) getInstanceTypeFromInstanceRequirements(imageId string, req start = time.Now() var instanceTypes []string - err = m.GetInstanceTypesFromInstanceRequirementsPages(requirementsInput, func(page *ec2.GetInstanceTypesFromInstanceRequirementsOutput, isLastPage bool) bool { + + // Use pagination + var nextToken *string + for { + if nextToken != nil { + requirementsInput.NextToken = nextToken + } + page, err := m.GetInstanceTypesFromInstanceRequirements(ctx, requirementsInput) + if err != nil { + observeAWSRequest("GetInstanceTypesFromInstanceRequirements", err, start) + return "", fmt.Errorf("unable to get instance types from requirements: %w", err) + } for _, instanceType := range page.InstanceTypes { instanceTypes = append(instanceTypes, *instanceType.InstanceType) } - return !isLastPage - }) - observeAWSRequest("GetInstanceTypesFromInstanceRequirements", err, start) - if err != nil { - return "", fmt.Errorf("unable to get instance types from requirements: %w", err) + nextToken = page.NextToken + if nextToken == nil { + break + } } + observeAWSRequest("GetInstanceTypesFromInstanceRequirements", nil, start) if len(instanceTypes) == 0 { return "", fmt.Errorf("no instance types found for requirements") @@ -348,16 +385,16 @@ func (m *awsWrapper) getInstanceTypeFromInstanceRequirements(imageId string, req return instanceTypes[0], nil } -func (m *awsWrapper) getRequirementsRequestFromAutoscaling(requirements *autoscaling.InstanceRequirements) (*ec2.InstanceRequirementsRequest, error) { - requirementsRequest := ec2.InstanceRequirementsRequest{} +func (m *awsWrapper) getRequirementsRequestFromAutoscaling(requirements *autoscalingtypes.InstanceRequirements) (*ec2types.InstanceRequirementsRequest, error) { + requirementsRequest := ec2types.InstanceRequirementsRequest{} // required instance requirements - requirementsRequest.MemoryMiB = &ec2.MemoryMiBRequest{ + requirementsRequest.MemoryMiB = &ec2types.MemoryMiBRequest{ Min: requirements.MemoryMiB.Min, Max: requirements.MemoryMiB.Max, } - requirementsRequest.VCpuCount = &ec2.VCpuCountRangeRequest{ + requirementsRequest.VCpuCount = &ec2types.VCpuCountRangeRequest{ Min: requirements.VCpuCount.Min, Max: requirements.VCpuCount.Max, } @@ -460,16 +497,16 @@ func (m *awsWrapper) getRequirementsRequestFromAutoscaling(requirements *autosca return &requirementsRequest, nil } -func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2.InstanceRequirements) (*ec2.InstanceRequirementsRequest, error) { - requirementsRequest := ec2.InstanceRequirementsRequest{} +func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2types.InstanceRequirements) (*ec2types.InstanceRequirementsRequest, error) { + requirementsRequest := ec2types.InstanceRequirementsRequest{} // required instance requirements - requirementsRequest.MemoryMiB = &ec2.MemoryMiBRequest{ + requirementsRequest.MemoryMiB = &ec2types.MemoryMiBRequest{ Min: requirements.MemoryMiB.Min, Max: requirements.MemoryMiB.Max, } - requirementsRequest.VCpuCount = &ec2.VCpuCountRangeRequest{ + requirementsRequest.VCpuCount = &ec2types.VCpuCountRangeRequest{ Min: requirements.VCpuCount.Min, Max: requirements.VCpuCount.Max, } @@ -572,11 +609,11 @@ func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2.InstanceReq return &requirementsRequest, nil } -func (m *awsWrapper) getEC2RequirementsFromAutoscaling(autoscalingRequirements *autoscaling.InstanceRequirements) (*ec2.InstanceRequirements, error) { - ec2Requirements := ec2.InstanceRequirements{} +func (m *awsWrapper) getEC2RequirementsFromAutoscaling(autoscalingRequirements *autoscalingtypes.InstanceRequirements) (*ec2types.InstanceRequirements, error) { + ec2Requirements := ec2types.InstanceRequirements{} // required instance requirements - ec2Requirements.MemoryMiB = &ec2.MemoryMiB{ + ec2Requirements.MemoryMiB = &ec2types.MemoryMiB{ Min: autoscalingRequirements.MemoryMiB.Min, Max: autoscalingRequirements.MemoryMiB.Max, } @@ -711,9 +748,9 @@ func (m *awsWrapper) getInstanceTypesForAsgs(asgs []*asg) (map[string]string, er klog.V(4).Infof("%d launch templates to query", len(launchTemplatesToQuery)) // Query these all at once to minimize AWS API calls - launchConfigNames := make([]*string, 0, len(launchConfigsToQuery)) + launchConfigNames := make([]string, 0, len(launchConfigsToQuery)) for _, cfgName := range launchConfigsToQuery { - launchConfigNames = append(launchConfigNames, aws.String(cfgName)) + launchConfigNames = append(launchConfigNames, cfgName) } launchConfigs, err := m.getInstanceTypeByLaunchConfigNames(launchConfigNames) if err != nil { @@ -755,7 +792,7 @@ func (m *awsWrapper) getInstanceTypesForAsgs(asgs []*asg) (map[string]string, er return results, nil } -func buildLaunchTemplateFromSpec(ltSpec *autoscaling.LaunchTemplateSpecification) *launchTemplate { +func buildLaunchTemplateFromSpec(ltSpec *autoscalingtypes.LaunchTemplateSpecification) *launchTemplate { // NOTE(jaypipes): The LaunchTemplateSpecification.Version is a pointer to // string. When the pointer is nil, EC2 AutoScaling API considers the value // to be "$Default", however aws.StringValue(ltSpec.Version) will return an @@ -785,18 +822,18 @@ func buildLaunchTemplateFromSpec(ltSpec *autoscaling.LaunchTemplateSpecification } } -func taintEksTranslator(t *eks.Taint) (apiv1.TaintEffect, error) { +func taintEksTranslator(t *ekstypes.Taint) (apiv1.TaintEffect, error) { // Translation between AWS EKS and Kubernetes taints // // See: // // https://docs.aws.amazon.com/eks/latest/APIReference/API_Taint.html - switch effect := *t.Effect; effect { - case eks.TaintEffectNoSchedule: + switch effect := t.Effect; effect { + case ekstypes.TaintEffectNoSchedule: return apiv1.TaintEffectNoSchedule, nil - case eks.TaintEffectNoExecute: + case ekstypes.TaintEffectNoExecute: return apiv1.TaintEffectNoExecute, nil - case eks.TaintEffectPreferNoSchedule: + case ekstypes.TaintEffectPreferNoSchedule: return apiv1.TaintEffectPreferNoSchedule, nil default: return "", fmt.Errorf("couldn't translate EKS DescribeNodegroup response taint %s into Kubernetes format", effect) diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index 5a58f093c4a4..c40f88ade94f 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -90,6 +90,22 @@ require ( github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect + github.com/aws/aws-sdk-go-v2 v1.39.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.31.13 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.18.17 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/service/autoscaling v1.59.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2 // indirect + github.com/aws/aws-sdk-go-v2/service/eks v1.74.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.29.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.38.7 // indirect + github.com/aws/smithy-go v1.23.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/cluster-autoscaler/go.sum b/cluster-autoscaler/go.sum index 8668deef6866..dd03e4e57faa 100644 --- a/cluster-autoscaler/go.sum +++ b/cluster-autoscaler/go.sum @@ -95,6 +95,38 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.44.241 h1:D3KycZq3HjhmjYGzvTcmX/Ztf/KNmsfTmdDuKdnzZKo= github.com/aws/aws-sdk-go v1.44.241/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.39.3 h1:h7xSsanJ4EQJXG5iuW4UqgP7qBopLpj84mpkNx3wPjM= +github.com/aws/aws-sdk-go-v2 v1.39.3/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM= +github.com/aws/aws-sdk-go-v2/config v1.31.13 h1:wcqQB3B0PgRPUF5ZE/QL1JVOyB0mbPevHFoAMpemR9k= +github.com/aws/aws-sdk-go-v2/config v1.31.13/go.mod h1:ySB5D5ybwqGbT6c3GszZ+u+3KvrlYCUQNo62+hkKOFk= +github.com/aws/aws-sdk-go-v2/credentials v1.18.17 h1:skpEwzN/+H8cdrrtT8y+rvWJGiWWv0DeNAe+4VTf+Vs= +github.com/aws/aws-sdk-go-v2/credentials v1.18.17/go.mod h1:Ed+nXsaYa5uBINovJhcAWkALvXw2ZLk36opcuiSZfJM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10 h1:UuGVOX48oP4vgQ36oiKmW9RuSeT8jlgQgBFQD+HUiHY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10/go.mod h1:vM/Ini41PzvudT4YkQyE/+WiQJiQ6jzeDyU8pQKwCac= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 h1:mj/bdWleWEh81DtpdHKkw41IrS+r3uw1J/VQtbwYYp8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10/go.mod h1:7+oEMxAZWP8gZCyjcm9VicI0M61Sx4DJtcGfKYv2yKQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 h1:wh+/mn57yhUrFtLIxyFPh2RgxgQz/u+Yrf7hiHGHqKY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10/go.mod h1:7zirD+ryp5gitJJ2m1BBux56ai8RIRDykXZrJSp540w= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.59.4 h1:nRvgLIy35Ls71GdH7RD/HJTfxCbdRuCLJFvARZOotic= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.59.4/go.mod h1:jgw3NI16z3l9PKxQUxpnkGZtkr8W1/eWiRJAcaZ5VSo= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2 h1:D8MCemFa8rt09x7o6Fkm2T7ThVbRPrD91R+LKhVEnVU= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2/go.mod h1:Q/kZ++hvhasMpQU37I7daQh07ZqTa++isjj1aPi4zvM= +github.com/aws/aws-sdk-go-v2/service/eks v1.74.3 h1:zdWTZYq9Sp1sTTXAMy/r6lHwXkzXg2V3GoH3Rn6FJlQ= +github.com/aws/aws-sdk-go-v2/service/eks v1.74.3/go.mod h1:o1FKzg3LHlNZP8p6mdFxzxPjJfmjww7WdX7EyVomXIo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 h1:DRND0dkCKtJzCj4Xl4OpVbXZgfttY5q712H9Zj7qc/0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10/go.mod h1:tGGNmJKOTernmR2+VJ0fCzQRurcPZj9ut60Zu5Fi6us= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.7 h1:fspVFg6qMx0svs40YgRmE7LZXh9VRZvTT35PfdQR6FM= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.7/go.mod h1:BQTKL3uMECaLaUV3Zc2L4Qybv8C6BIXjuu1dOPyxTQs= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2 h1:scVnW+NLXasGOhy7HhkdT9AGb6kjgW7fJ5xYkUaqHs0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2/go.mod h1:FRNCY3zTEWZXBKm2h5UBUPvCVDOecTad9KhynDyGBc0= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.7 h1:VEO5dqFkMsl8QZ2yHsFDJAIZLAkEbaYDB+xdKi0Feic= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.7/go.mod h1:L1xxV3zAdB+qVrVW/pBIrIAnHFWHo6FBbFe4xOGsG/o= +github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M= +github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=