From 2f230b38348af971c2bb4f6214d620f8774f1e1a Mon Sep 17 00:00:00 2001 From: Marco Braga Date: Fri, 3 May 2024 19:42:58 -0300 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Support=20BYO=20Public=20IPv4=20Poo?= =?UTF-8?q?l=20for=20Elastic=20IPs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introducing support of BYO Public IPv4 Pool to allow CAPA allocate IPv4 Elastic IPs from user-provided IPv4 pools that was brought to AWS when provisioning base cluster infrastructure. This change introduce the API fields: - AWSCLuster NetworkSpec.ElasticIPPool: allowing the controllers to consume from user-provided public pools when provisioning core components from the infrastructure, like Nat Gateways and public Network Load Balancer (API server only) - AWSMachine ElasticIPPool: allowing the machine to consume from BYO Public IPv4 pool when the instance is deployed in the public subnets. The ElasticIPPool structure defines a custom IPv4 Pool (previously created in the AWS Account) to teach controllers to set the pool when creating public ip addresses (Elastic IPs) for components which requires it, such as Nat Gateways and NLBs. --- api/v1beta1/awscluster_conversion.go | 12 ++ api/v1beta1/awsmachine_conversion.go | 22 +++ api/v1beta1/zz_generated.conversion.go | 2 + api/v1beta2/awscluster_webhook.go | 16 ++ api/v1beta2/awsmachine_types.go | 5 + api/v1beta2/awsmachine_webhook.go | 27 +++ api/v1beta2/awsmachine_webhook_test.go | 68 +++++++ api/v1beta2/network_types.go | 76 ++++++++ api/v1beta2/zz_generated.deepcopy.go | 35 ++++ ...ster.x-k8s.io_awsmanagedcontrolplanes.yaml | 66 +++++++ ...tructure.cluster.x-k8s.io_awsclusters.yaml | 33 ++++ ....cluster.x-k8s.io_awsclustertemplates.yaml | 33 ++++ ...tructure.cluster.x-k8s.io_awsmachines.yaml | 31 +++ ....cluster.x-k8s.io_awsmachinetemplates.yaml | 31 +++ controllers/awscluster_controller_test.go | 17 +- .../awscluster_controller_unit_test.go | 32 +++ controllers/awsmachine_controller.go | 24 +++ controllers/awsmachine_controller_test.go | 4 + .../awsmachine_controller_unit_test.go | 35 ++++ .../awsmanagedcontrolplane_controller_test.go | 6 +- .../bring-your-own-aws-infrastructure.md | 66 +++++++ pkg/cloud/scope/machine.go | 8 + pkg/cloud/services/ec2/eip.go | 54 ++++++ pkg/cloud/services/ec2/instances.go | 21 +- pkg/cloud/services/ec2/instances_test.go | 74 ------- pkg/cloud/services/ec2/service.go | 13 +- pkg/cloud/services/elb/eip.go | 55 ++++++ pkg/cloud/services/elb/loadbalancer.go | 16 ++ pkg/cloud/services/elb/service.go | 3 + pkg/cloud/services/interfaces.go | 5 + .../mock_services/ec2_interface_mock.go | 28 +++ pkg/cloud/services/network/eips.go | 183 ++++++++++++++---- pkg/cloud/services/network/natgateways.go | 2 +- .../services/network/natgateways_test.go | 8 +- 34 files changed, 966 insertions(+), 145 deletions(-) create mode 100644 pkg/cloud/services/ec2/eip.go create mode 100644 pkg/cloud/services/elb/eip.go diff --git a/api/v1beta1/awscluster_conversion.go b/api/v1beta1/awscluster_conversion.go index 382a4cd4d3..65e797ffba 100644 --- a/api/v1beta1/awscluster_conversion.go +++ b/api/v1beta1/awscluster_conversion.go @@ -105,6 +105,18 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch = restored.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch dst.Spec.NetworkSpec.VPC.CarrierGatewayID = restored.Spec.NetworkSpec.VPC.CarrierGatewayID + if restored.Spec.NetworkSpec.VPC.ElasticIPPool != nil { + if dst.Spec.NetworkSpec.VPC.ElasticIPPool == nil { + dst.Spec.NetworkSpec.VPC.ElasticIPPool = &infrav2.ElasticIPPool{} + } + if restored.Spec.NetworkSpec.VPC.ElasticIPPool.PublicIpv4Pool != nil { + dst.Spec.NetworkSpec.VPC.ElasticIPPool.PublicIpv4Pool = restored.Spec.NetworkSpec.VPC.ElasticIPPool.PublicIpv4Pool + } + if restored.Spec.NetworkSpec.VPC.ElasticIPPool.PublicIpv4PoolFallBackOrder != nil { + dst.Spec.NetworkSpec.VPC.ElasticIPPool.PublicIpv4PoolFallBackOrder = restored.Spec.NetworkSpec.VPC.ElasticIPPool.PublicIpv4PoolFallBackOrder + } + } + // Restore SubnetSpec.ResourceID, SubnetSpec.ParentZoneName, and SubnetSpec.ZoneType fields, if any. for _, subnet := range restored.Spec.NetworkSpec.Subnets { for i, dstSubnet := range dst.Spec.NetworkSpec.Subnets { diff --git a/api/v1beta1/awsmachine_conversion.go b/api/v1beta1/awsmachine_conversion.go index 3cd84b20a9..6ed2d50adc 100644 --- a/api/v1beta1/awsmachine_conversion.go +++ b/api/v1beta1/awsmachine_conversion.go @@ -41,6 +41,17 @@ func (src *AWSMachine) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.PlacementGroupPartition = restored.Spec.PlacementGroupPartition dst.Spec.PrivateDNSName = restored.Spec.PrivateDNSName dst.Spec.SecurityGroupOverrides = restored.Spec.SecurityGroupOverrides + if restored.Spec.ElasticIPPool != nil { + if dst.Spec.ElasticIPPool == nil { + dst.Spec.ElasticIPPool = &infrav1.ElasticIPPool{} + } + if restored.Spec.ElasticIPPool.PublicIpv4Pool != nil { + dst.Spec.ElasticIPPool.PublicIpv4Pool = restored.Spec.ElasticIPPool.PublicIpv4Pool + } + if restored.Spec.ElasticIPPool.PublicIpv4PoolFallBackOrder != nil { + dst.Spec.ElasticIPPool.PublicIpv4PoolFallBackOrder = restored.Spec.ElasticIPPool.PublicIpv4PoolFallBackOrder + } + } return nil } @@ -91,6 +102,17 @@ func (r *AWSMachineTemplate) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Template.Spec.PlacementGroupPartition = restored.Spec.Template.Spec.PlacementGroupPartition dst.Spec.Template.Spec.PrivateDNSName = restored.Spec.Template.Spec.PrivateDNSName dst.Spec.Template.Spec.SecurityGroupOverrides = restored.Spec.Template.Spec.SecurityGroupOverrides + if restored.Spec.Template.Spec.ElasticIPPool != nil { + if dst.Spec.Template.Spec.ElasticIPPool == nil { + dst.Spec.Template.Spec.ElasticIPPool = &infrav1.ElasticIPPool{} + } + if restored.Spec.Template.Spec.ElasticIPPool.PublicIpv4Pool != nil { + dst.Spec.Template.Spec.ElasticIPPool.PublicIpv4Pool = restored.Spec.Template.Spec.ElasticIPPool.PublicIpv4Pool + } + if restored.Spec.Template.Spec.ElasticIPPool.PublicIpv4PoolFallBackOrder != nil { + dst.Spec.Template.Spec.ElasticIPPool.PublicIpv4PoolFallBackOrder = restored.Spec.Template.Spec.ElasticIPPool.PublicIpv4PoolFallBackOrder + } + } return nil } diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 10842bb9ae..88a436a128 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -1389,6 +1389,7 @@ func autoConvert_v1beta2_AWSMachineSpec_To_v1beta1_AWSMachineSpec(in *v1beta2.AW out.AdditionalTags = *(*Tags)(unsafe.Pointer(&in.AdditionalTags)) out.IAMInstanceProfile = in.IAMInstanceProfile out.PublicIP = (*bool)(unsafe.Pointer(in.PublicIP)) + // WARNING: in.ElasticIPPool requires manual conversion: does not exist in peer-type if in.AdditionalSecurityGroups != nil { in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups *out = make([]AWSResourceReference, len(*in)) @@ -2313,6 +2314,7 @@ func autoConvert_v1beta2_VPCSpec_To_v1beta1_VPCSpec(in *v1beta2.VPCSpec, out *VP out.AvailabilityZoneSelection = (*AZSelectionScheme)(unsafe.Pointer(in.AvailabilityZoneSelection)) // WARNING: in.EmptyRoutesDefaultVPCSecurityGroup requires manual conversion: does not exist in peer-type // WARNING: in.PrivateDNSHostnameTypeOnLaunch requires manual conversion: does not exist in peer-type + // WARNING: in.ElasticIPPool requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1beta2/awscluster_webhook.go b/api/v1beta2/awscluster_webhook.go index ae9c80f5b4..b9661844f2 100644 --- a/api/v1beta2/awscluster_webhook.go +++ b/api/v1beta2/awscluster_webhook.go @@ -269,6 +269,22 @@ func (r *AWSCluster) validateNetwork() field.ErrorList { } } + if r.Spec.NetworkSpec.VPC.ElasticIPPool != nil { + eipp := r.Spec.NetworkSpec.VPC.ElasticIPPool + if eipp.PublicIpv4Pool != nil { + if eipp.PublicIpv4PoolFallBackOrder == nil { + return append(allErrs, field.Invalid(field.NewPath("elasticIpPool.publicIpv4PoolFallbackOrder"), r.Spec.NetworkSpec.VPC.ElasticIPPool, "publicIpv4PoolFallbackOrder must be set when publicIpv4Pool is defined.")) + } + awsPublicIpv4PoolPrefix := "ipv4pool-ec2-" + if !strings.HasPrefix(*eipp.PublicIpv4Pool, awsPublicIpv4PoolPrefix) { + return append(allErrs, field.Invalid(field.NewPath("elasticIpPool.publicIpv4Pool"), r.Spec.NetworkSpec.VPC.ElasticIPPool, fmt.Sprintf("publicIpv4Pool must start with %s.", awsPublicIpv4PoolPrefix))) + } + } + if eipp.PublicIpv4Pool == nil && eipp.PublicIpv4PoolFallBackOrder != nil { + return append(allErrs, field.Invalid(field.NewPath("elasticIpPool.publicIpv4PoolFallbackOrder"), r.Spec.NetworkSpec.VPC.ElasticIPPool, "publicIpv4Pool must be set when publicIpv4PoolFallbackOrder is defined.")) + } + } + return allErrs } diff --git a/api/v1beta2/awsmachine_types.go b/api/v1beta2/awsmachine_types.go index 26e733b9c5..1a527f9991 100644 --- a/api/v1beta2/awsmachine_types.go +++ b/api/v1beta2/awsmachine_types.go @@ -113,6 +113,11 @@ type AWSMachineSpec struct { // +optional PublicIP *bool `json:"publicIP,omitempty"` + // ElasticIPPool is the configuration to allocate Public IPv4 address (Elastic IP/EIP) from user-defined pool. + // + // +optional + ElasticIPPool *ElasticIPPool `json:"elasticIpPool,omitempty"` + // AdditionalSecurityGroups is an array of references to security groups that should be applied to the // instance. These security groups would be set in addition to any security groups defined // at the cluster level or in the actuator. It is possible to specify either IDs of Filters. Using Filters diff --git a/api/v1beta2/awsmachine_webhook.go b/api/v1beta2/awsmachine_webhook.go index 8938e01dfb..50af4f2211 100644 --- a/api/v1beta2/awsmachine_webhook.go +++ b/api/v1beta2/awsmachine_webhook.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -64,6 +65,7 @@ func (r *AWSMachine) ValidateCreate() (admission.Warnings, error) { allErrs = append(allErrs, r.validateSSHKeyName()...) allErrs = append(allErrs, r.validateAdditionalSecurityGroups()...) allErrs = append(allErrs, r.Spec.AdditionalTags.Validate()...) + allErrs = append(allErrs, r.validateNetworkElasticIPPool()...) return nil, aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs) } @@ -334,6 +336,31 @@ func (r *AWSMachine) validateRootVolume() field.ErrorList { return allErrs } +func (r *AWSMachine) validateNetworkElasticIPPool() field.ErrorList { + var allErrs field.ErrorList + + if r.Spec.ElasticIPPool == nil { + return allErrs + } + if !ptr.Deref(r.Spec.PublicIP, false) { + allErrs = append(allErrs, field.Required(field.NewPath("spec.elasticIpPool"), "publicIp must be set to 'true' to assign custom public IPv4 pools with elasticIpPool")) + } + eipp := r.Spec.ElasticIPPool + if eipp.PublicIpv4Pool != nil { + if eipp.PublicIpv4PoolFallBackOrder == nil { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec.elasticIpPool.publicIpv4PoolFallbackOrder"), r.Spec.ElasticIPPool, "publicIpv4PoolFallbackOrder must be set when publicIpv4Pool is defined.")) + } + awsPublicIpv4PoolPrefix := "ipv4pool-ec2-" + if !strings.HasPrefix(*eipp.PublicIpv4Pool, awsPublicIpv4PoolPrefix) { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec.elasticIpPool.publicIpv4Pool"), r.Spec.ElasticIPPool, fmt.Sprintf("publicIpv4Pool must start with %s.", awsPublicIpv4PoolPrefix))) + } + } else if eipp.PublicIpv4PoolFallBackOrder != nil { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec.elasticIpPool.publicIpv4PoolFallbackOrder"), r.Spec.ElasticIPPool, "publicIpv4Pool must be set when publicIpv4PoolFallbackOrder is defined.")) + } + + return allErrs +} + func (r *AWSMachine) validateNonRootVolumes() field.ErrorList { var allErrs field.ErrorList diff --git a/api/v1beta2/awsmachine_webhook_test.go b/api/v1beta2/awsmachine_webhook_test.go index 8588211aa7..80e7abc45a 100644 --- a/api/v1beta2/awsmachine_webhook_test.go +++ b/api/v1beta2/awsmachine_webhook_test.go @@ -368,6 +368,74 @@ func TestAWSMachineCreate(t *testing.T) { }, wantErr: true, }, + { + name: "create with valid BYOIPv4", + machine: &AWSMachine{ + Spec: AWSMachineSpec{ + InstanceType: "type", + PublicIP: aws.Bool(true), + ElasticIPPool: &ElasticIPPool{ + PublicIpv4Pool: aws.String("ipv4pool-ec2-0123456789abcdef0"), + PublicIpv4PoolFallBackOrder: ptr.To(PublicIpv4PoolFallbackOrderAmazonPool), + }, + }, + }, + wantErr: false, + }, + { + name: "error when BYOIPv4 without fallback", + machine: &AWSMachine{ + Spec: AWSMachineSpec{ + InstanceType: "type", + PublicIP: aws.Bool(true), + ElasticIPPool: &ElasticIPPool{ + PublicIpv4Pool: aws.String("ipv4pool-ec2-0123456789abcdef0"), + }, + }, + }, + wantErr: true, + }, + { + name: "error when BYOIPv4 without public ipv4 pool", + machine: &AWSMachine{ + Spec: AWSMachineSpec{ + InstanceType: "type", + PublicIP: aws.Bool(true), + ElasticIPPool: &ElasticIPPool{ + PublicIpv4PoolFallBackOrder: ptr.To(PublicIpv4PoolFallbackOrderAmazonPool), + }, + }, + }, + wantErr: true, + }, + { + name: "error when BYOIPv4 with non-public IP set", + machine: &AWSMachine{ + Spec: AWSMachineSpec{ + InstanceType: "type", + PublicIP: aws.Bool(false), + ElasticIPPool: &ElasticIPPool{ + PublicIpv4Pool: aws.String("ipv4pool-ec2-0123456789abcdef0"), + PublicIpv4PoolFallBackOrder: ptr.To(PublicIpv4PoolFallbackOrderAmazonPool), + }, + }, + }, + wantErr: true, + }, + { + name: "error when BYOIPv4 with invalid pool name", + machine: &AWSMachine{ + Spec: AWSMachineSpec{ + InstanceType: "type", + PublicIP: aws.Bool(true), + ElasticIPPool: &ElasticIPPool{ + PublicIpv4Pool: aws.String("ipv4poolx-ec2-0123456789abcdef"), + PublicIpv4PoolFallBackOrder: ptr.To(PublicIpv4PoolFallbackOrderAmazonPool), + }, + }, + }, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/api/v1beta2/network_types.go b/api/v1beta2/network_types.go index 83b5d3a7d1..2a0050156c 100644 --- a/api/v1beta2/network_types.go +++ b/api/v1beta2/network_types.go @@ -455,6 +455,12 @@ type VPCSpec struct { // +optional // +kubebuilder:validation:Enum:=ip-name;resource-name PrivateDNSHostnameTypeOnLaunch *string `json:"privateDnsHostnameTypeOnLaunch,omitempty"` + + // ElasticIPPool contains specific configuration to allocate Public IPv4 address (Elastic IP) from user-defined pool + // brought to AWS for core infrastructure resources, like NAT Gateways and Public Network Load Balancers for + // the API Server. + // +optional + ElasticIPPool *ElasticIPPool `json:"elasticIpPool,omitempty"` } // String returns a string representation of the VPC. @@ -477,6 +483,22 @@ func (v *VPCSpec) IsIPv6Enabled() bool { return v.IPv6 != nil } +// GetElasticIPPool returns the custom Elastic IP Pool configuration when present. +func (v *VPCSpec) GetElasticIPPool() *ElasticIPPool { + return v.ElasticIPPool +} + +// GetPublicIpv4Pool returns the custom public IPv4 pool brought to AWS when present. +func (v *VPCSpec) GetPublicIpv4Pool() *string { + if v.ElasticIPPool == nil { + return nil + } + if v.ElasticIPPool.PublicIpv4Pool != nil { + return v.ElasticIPPool.PublicIpv4Pool + } + return nil +} + // SubnetSpec configures an AWS Subnet. type SubnetSpec struct { // ID defines a unique identifier to reference this resource. @@ -1013,3 +1035,57 @@ func (z ZoneType) String() string { func (z ZoneType) Equal(other ZoneType) bool { return z == other } + +// ElasticIPPool allows configuring a Elastic IP pool for resources allocating +// public IPv4 addresses on public subnets. +type ElasticIPPool struct { + // PublicIpv4Pool sets a custom Public IPv4 Pool used to create Elastic IP address for resources + // created in public IPv4 subnets. Every IPv4 address, Elastic IP, will be allocated from the custom + // Public IPv4 pool that you brought to AWS, instead of Amazon-provided pool. The public IPv4 pool + // resource ID starts with 'ipv4pool-ec2'. + // + // +kubebuilder:validation:MaxLength=30 + // +optional + PublicIpv4Pool *string `json:"publicIpv4Pool,omitempty"` + + // PublicIpv4PoolFallBackOrder defines the fallback action when the Public IPv4 Pool has been exhausted, + // no more IPv4 address available in the pool. + // + // When set to 'amazon-pool', the controller check if the pool has available IPv4 address, when pool has reached the + // IPv4 limit, the address will be claimed from Amazon-pool (default). + // + // When set to 'none', the controller will fail the Elastic IP allocation when the publicIpv4Pool is exhausted. + // + // +kubebuilder:validation:Enum:=amazon-pool;none + // +optional + PublicIpv4PoolFallBackOrder *PublicIpv4PoolFallbackOrder `json:"publicIpv4PoolFallbackOrder,omitempty"` + + // TODO(mtulio): add future support of user-defined Elastic IP to allow users to assign BYO Public IP from + // 'static'/preallocated amazon-provided IPsstrucute currently holds only 'BYO Public IP from Public IPv4 Pool' (user brought to AWS), + // although a dedicated structure would help to hold 'BYO Elastic IP' variants like: + // - AllocationIdPoolApiLoadBalancer: an user-defined (static) IP address to the Public API Load Balancer. + // - AllocationIdPoolNatGateways: an user-defined (static) IP address to allocate to NAT Gateways (egress traffic). +} + +// PublicIpv4PoolFallbackOrder defines the list of available fallback action when the PublicIpv4Pool is exhausted. +// 'none' let the controllers return failures when the PublicIpv4Pool is exhausted - no more IPv4 available. +// 'amazon-pool' let the controllers to skip the PublicIpv4Pool and use the Amazon pool, the default. +// +kubebuilder:validation:XValidation:rule="self in ['none','amazon-pool']",message="allowed values are 'none' and 'amazon-pool'" +type PublicIpv4PoolFallbackOrder string + +const ( + // PublicIpv4PoolFallbackOrderAmazonPool refers to use Amazon-pool Public IPv4 Pool as a fallback strategy. + PublicIpv4PoolFallbackOrderAmazonPool = PublicIpv4PoolFallbackOrder("amazon-pool") + + // PublicIpv4PoolFallbackOrderNone refers to not use any fallback strategy. + PublicIpv4PoolFallbackOrderNone = PublicIpv4PoolFallbackOrder("none") +) + +func (r PublicIpv4PoolFallbackOrder) String() string { + return string(r) +} + +// Equal compares PublicIpv4PoolFallbackOrder types and return true if input param is equal. +func (r PublicIpv4PoolFallbackOrder) Equal(e PublicIpv4PoolFallbackOrder) bool { + return r == e +} diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 81b8a8d314..d3e08bbb0c 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -700,6 +700,11 @@ func (in *AWSMachineSpec) DeepCopyInto(out *AWSMachineSpec) { *out = new(bool) **out = **in } + if in.ElasticIPPool != nil { + in, out := &in.ElasticIPPool, &out.ElasticIPPool + *out = new(ElasticIPPool) + (*in).DeepCopyInto(*out) + } if in.AdditionalSecurityGroups != nil { in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups *out = make([]AWSResourceReference, len(*in)) @@ -1281,6 +1286,31 @@ func (in *CloudInit) DeepCopy() *CloudInit { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ElasticIPPool) DeepCopyInto(out *ElasticIPPool) { + *out = *in + if in.PublicIpv4Pool != nil { + in, out := &in.PublicIpv4Pool, &out.PublicIpv4Pool + *out = new(string) + **out = **in + } + if in.PublicIpv4PoolFallBackOrder != nil { + in, out := &in.PublicIpv4PoolFallBackOrder, &out.PublicIpv4PoolFallBackOrder + *out = new(PublicIpv4PoolFallbackOrder) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ElasticIPPool. +func (in *ElasticIPPool) DeepCopy() *ElasticIPPool { + if in == nil { + return nil + } + out := new(ElasticIPPool) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Filter) DeepCopyInto(out *Filter) { *out = *in @@ -2157,6 +2187,11 @@ func (in *VPCSpec) DeepCopyInto(out *VPCSpec) { *out = new(string) **out = **in } + if in.ElasticIPPool != nil { + in, out := &in.ElasticIPPool, &out.ElasticIPPool + *out = new(ElasticIPPool) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCSpec. diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index c9ffb5ecd8..0fe39dcdd7 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -628,6 +628,39 @@ spec: Defaults to 10.0.0.0/16. Mutually exclusive with IPAMPool. type: string + elasticIpPool: + description: |- + ElasticIPPool contains specific configuration to allocate Public IPv4 address (Elastic IP) from user-defined pool + brought to AWS for core infrastructure resources, like NAT Gateways and Public Network Load Balancers for + the API Server. + properties: + publicIpv4Pool: + description: |- + PublicIpv4Pool sets a custom Public IPv4 Pool used to create Elastic IP address for resources + created in public IPv4 subnets. Every IPv4 address, Elastic IP, will be allocated from the custom + Public IPv4 pool that you brought to AWS, instead of Amazon-provided pool. The public IPv4 pool + resource ID starts with 'ipv4pool-ec2'. + maxLength: 30 + type: string + publicIpv4PoolFallbackOrder: + description: |- + PublicIpv4PoolFallBackOrder defines the fallback action when the Public IPv4 Pool has been exhausted, + no more IPv4 address available in the pool. + + + When set to 'amazon-pool', the controller check if the pool has available IPv4 address, when pool has reached the + IPv4 limit, the address will be claimed from Amazon-pool (default). + + + When set to 'none', the controller will fail the Elastic IP allocation when the publicIpv4Pool is exhausted. + enum: + - amazon-pool + - none + type: string + x-kubernetes-validations: + - message: allowed values are 'none' and 'amazon-pool' + rule: self in ['none','amazon-pool'] + type: object emptyRoutesDefaultVPCSecurityGroup: description: |- EmptyRoutesDefaultVPCSecurityGroup specifies whether the default VPC security group ingress @@ -2578,6 +2611,39 @@ spec: Defaults to 10.0.0.0/16. Mutually exclusive with IPAMPool. type: string + elasticIpPool: + description: |- + ElasticIPPool contains specific configuration to allocate Public IPv4 address (Elastic IP) from user-defined pool + brought to AWS for core infrastructure resources, like NAT Gateways and Public Network Load Balancers for + the API Server. + properties: + publicIpv4Pool: + description: |- + PublicIpv4Pool sets a custom Public IPv4 Pool used to create Elastic IP address for resources + created in public IPv4 subnets. Every IPv4 address, Elastic IP, will be allocated from the custom + Public IPv4 pool that you brought to AWS, instead of Amazon-provided pool. The public IPv4 pool + resource ID starts with 'ipv4pool-ec2'. + maxLength: 30 + type: string + publicIpv4PoolFallbackOrder: + description: |- + PublicIpv4PoolFallBackOrder defines the fallback action when the Public IPv4 Pool has been exhausted, + no more IPv4 address available in the pool. + + + When set to 'amazon-pool', the controller check if the pool has available IPv4 address, when pool has reached the + IPv4 limit, the address will be claimed from Amazon-pool (default). + + + When set to 'none', the controller will fail the Elastic IP allocation when the publicIpv4Pool is exhausted. + enum: + - amazon-pool + - none + type: string + x-kubernetes-validations: + - message: allowed values are 'none' and 'amazon-pool' + rule: self in ['none','amazon-pool'] + type: object emptyRoutesDefaultVPCSecurityGroup: description: |- EmptyRoutesDefaultVPCSecurityGroup specifies whether the default VPC security group ingress diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index f2d4b882b5..ec0208aad7 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -1564,6 +1564,39 @@ spec: Defaults to 10.0.0.0/16. Mutually exclusive with IPAMPool. type: string + elasticIpPool: + description: |- + ElasticIPPool contains specific configuration to allocate Public IPv4 address (Elastic IP) from user-defined pool + brought to AWS for core infrastructure resources, like NAT Gateways and Public Network Load Balancers for + the API Server. + properties: + publicIpv4Pool: + description: |- + PublicIpv4Pool sets a custom Public IPv4 Pool used to create Elastic IP address for resources + created in public IPv4 subnets. Every IPv4 address, Elastic IP, will be allocated from the custom + Public IPv4 pool that you brought to AWS, instead of Amazon-provided pool. The public IPv4 pool + resource ID starts with 'ipv4pool-ec2'. + maxLength: 30 + type: string + publicIpv4PoolFallbackOrder: + description: |- + PublicIpv4PoolFallBackOrder defines the fallback action when the Public IPv4 Pool has been exhausted, + no more IPv4 address available in the pool. + + + When set to 'amazon-pool', the controller check if the pool has available IPv4 address, when pool has reached the + IPv4 limit, the address will be claimed from Amazon-pool (default). + + + When set to 'none', the controller will fail the Elastic IP allocation when the publicIpv4Pool is exhausted. + enum: + - amazon-pool + - none + type: string + x-kubernetes-validations: + - message: allowed values are 'none' and 'amazon-pool' + rule: self in ['none','amazon-pool'] + type: object emptyRoutesDefaultVPCSecurityGroup: description: |- EmptyRoutesDefaultVPCSecurityGroup specifies whether the default VPC security group ingress diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml index ccc966dbb2..b36017a8c6 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml @@ -1162,6 +1162,39 @@ spec: Defaults to 10.0.0.0/16. Mutually exclusive with IPAMPool. type: string + elasticIpPool: + description: |- + ElasticIPPool contains specific configuration to allocate Public IPv4 address (Elastic IP) from user-defined pool + brought to AWS for core infrastructure resources, like NAT Gateways and Public Network Load Balancers for + the API Server. + properties: + publicIpv4Pool: + description: |- + PublicIpv4Pool sets a custom Public IPv4 Pool used to create Elastic IP address for resources + created in public IPv4 subnets. Every IPv4 address, Elastic IP, will be allocated from the custom + Public IPv4 pool that you brought to AWS, instead of Amazon-provided pool. The public IPv4 pool + resource ID starts with 'ipv4pool-ec2'. + maxLength: 30 + type: string + publicIpv4PoolFallbackOrder: + description: |- + PublicIpv4PoolFallBackOrder defines the fallback action when the Public IPv4 Pool has been exhausted, + no more IPv4 address available in the pool. + + + When set to 'amazon-pool', the controller check if the pool has available IPv4 address, when pool has reached the + IPv4 limit, the address will be claimed from Amazon-pool (default). + + + When set to 'none', the controller will fail the Elastic IP allocation when the publicIpv4Pool is exhausted. + enum: + - amazon-pool + - none + type: string + x-kubernetes-validations: + - message: allowed values are 'none' and 'amazon-pool' + rule: self in ['none','amazon-pool'] + type: object emptyRoutesDefaultVPCSecurityGroup: description: |- EmptyRoutesDefaultVPCSecurityGroup specifies whether the default VPC security group ingress diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml index c16031df5d..5b3aba0f22 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml @@ -657,6 +657,37 @@ spec: - ssm-parameter-store type: string type: object + elasticIpPool: + description: ElasticIPPool is the configuration to allocate Public + IPv4 address (Elastic IP/EIP) from user-defined pool. + properties: + publicIpv4Pool: + description: |- + PublicIpv4Pool sets a custom Public IPv4 Pool used to create Elastic IP address for resources + created in public IPv4 subnets. Every IPv4 address, Elastic IP, will be allocated from the custom + Public IPv4 pool that you brought to AWS, instead of Amazon-provided pool. The public IPv4 pool + resource ID starts with 'ipv4pool-ec2'. + maxLength: 30 + type: string + publicIpv4PoolFallbackOrder: + description: |- + PublicIpv4PoolFallBackOrder defines the fallback action when the Public IPv4 Pool has been exhausted, + no more IPv4 address available in the pool. + + + When set to 'amazon-pool', the controller check if the pool has available IPv4 address, when pool has reached the + IPv4 limit, the address will be claimed from Amazon-pool (default). + + + When set to 'none', the controller will fail the Elastic IP allocation when the publicIpv4Pool is exhausted. + enum: + - amazon-pool + - none + type: string + x-kubernetes-validations: + - message: allowed values are 'none' and 'amazon-pool' + rule: self in ['none','amazon-pool'] + type: object iamInstanceProfile: description: IAMInstanceProfile is a name of an IAM instance profile to assign to the instance diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml index c824b910db..fc97c0757b 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml @@ -587,6 +587,37 @@ spec: - ssm-parameter-store type: string type: object + elasticIpPool: + description: ElasticIPPool is the configuration to allocate + Public IPv4 address (Elastic IP/EIP) from user-defined pool. + properties: + publicIpv4Pool: + description: |- + PublicIpv4Pool sets a custom Public IPv4 Pool used to create Elastic IP address for resources + created in public IPv4 subnets. Every IPv4 address, Elastic IP, will be allocated from the custom + Public IPv4 pool that you brought to AWS, instead of Amazon-provided pool. The public IPv4 pool + resource ID starts with 'ipv4pool-ec2'. + maxLength: 30 + type: string + publicIpv4PoolFallbackOrder: + description: |- + PublicIpv4PoolFallBackOrder defines the fallback action when the Public IPv4 Pool has been exhausted, + no more IPv4 address available in the pool. + + + When set to 'amazon-pool', the controller check if the pool has available IPv4 address, when pool has reached the + IPv4 limit, the address will be claimed from Amazon-pool (default). + + + When set to 'none', the controller will fail the Elastic IP allocation when the publicIpv4Pool is exhausted. + enum: + - amazon-pool + - none + type: string + x-kubernetes-validations: + - message: allowed values are 'none' and 'amazon-pool' + rule: self in ['none','amazon-pool'] + type: object iamInstanceProfile: description: IAMInstanceProfile is a name of an IAM instance profile to assign to the instance diff --git a/controllers/awscluster_controller_test.go b/controllers/awscluster_controller_test.go index 828bfae822..93d139a5a0 100644 --- a/controllers/awscluster_controller_test.go +++ b/controllers/awscluster_controller_test.go @@ -1206,7 +1206,7 @@ func mockedCallsForMissingEverything(m *mocks.MockEC2APIMockRecorder, e *mocks.M }, { Name: aws.String("tag:sigs.k8s.io/cluster-api-provider-aws/role"), - Values: aws.StringSlice([]string{"apiserver"}), + Values: aws.StringSlice([]string{"common"}), }, }, })).Return(&ec2.DescribeAddressesOutput{ @@ -1221,7 +1221,7 @@ func mockedCallsForMissingEverything(m *mocks.MockEC2APIMockRecorder, e *mocks.M Tags: []*ec2.Tag{ { Key: aws.String("Name"), - Value: aws.String("test-cluster-eip-apiserver"), + Value: aws.String("test-cluster-eip-common"), }, { Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), @@ -1229,7 +1229,7 @@ func mockedCallsForMissingEverything(m *mocks.MockEC2APIMockRecorder, e *mocks.M }, { Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), - Value: aws.String("apiserver"), + Value: aws.String("common"), }, }, }, @@ -1459,7 +1459,12 @@ func mockedDeleteVPCCallsForNonExistentVPC(m *mocks.MockEC2APIMockRecorder) { { Name: aws.String("tag-key"), Values: aws.StringSlice([]string{"sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"}), - }}, + }, + { + Name: aws.String("tag:sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Values: aws.StringSlice([]string{"owned"}), + }, + }, })).Return(nil, nil) m.DeleteVpcWithContext(context.TODO(), gomock.AssignableToTypeOf(&ec2.DeleteVpcInput{ VpcId: aws.String("vpc-exists")})).Return(nil, nil) @@ -1550,6 +1555,10 @@ func mockedDeleteVPCCalls(m *mocks.MockEC2APIMockRecorder) { { Name: aws.String("tag-key"), Values: aws.StringSlice([]string{"sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"}), + }, + { + Name: aws.String("tag:sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Values: aws.StringSlice([]string{"owned"}), }}, })).Return(&ec2.DescribeAddressesOutput{ Addresses: []*ec2.Address{ diff --git a/controllers/awscluster_controller_unit_test.go b/controllers/awscluster_controller_unit_test.go index 9628ccb579..e282d2bf79 100644 --- a/controllers/awscluster_controller_unit_test.go +++ b/controllers/awscluster_controller_unit_test.go @@ -23,12 +23,14 @@ import ( "testing" "time" + "github.com/aws/aws-sdk-go/aws" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -247,6 +249,36 @@ func TestAWSClusterReconcileOperations(t *testing.T) { expectAWSClusterConditions(g, cs.AWSCluster, []conditionAssertion{{infrav1.LoadBalancerReadyCondition, corev1.ConditionTrue, "", ""}}) g.Expect(awsCluster.GetFinalizers()).To(ContainElement(infrav1.ClusterFinalizer)) }) + + t.Run("when BYO IP is set", func(t *testing.T) { + g := NewWithT(t) + runningCluster := func() { + ec2Svc.EXPECT().ReconcileBastion().Return(nil) + elbSvc.EXPECT().ReconcileLoadbalancers().Return(nil) + networkSvc.EXPECT().ReconcileNetwork().Return(nil) + sgSvc.EXPECT().ReconcileSecurityGroups().Return(nil) + } + + awsCluster := getAWSCluster("test", "test") + csClient := setup(t, &awsCluster) + defer teardown() + runningCluster() + cs, err := scope.NewClusterScope( + scope.ClusterScopeParams{ + Client: csClient, + Cluster: &clusterv1.Cluster{}, + AWSCluster: &awsCluster, + }, + ) + g.Expect(err).To(BeNil()) + awsCluster.Spec.NetworkSpec.VPC.ElasticIPPool = &infrav1.ElasticIPPool{ + PublicIpv4Pool: aws.String("ipv4pool-ec2-0123456789abcdef0"), + PublicIpv4PoolFallBackOrder: ptr.To(infrav1.PublicIpv4PoolFallbackOrderAmazonPool), + } + g.Expect(err).To(Not(HaveOccurred())) + _, err = reconciler.reconcileNormal(cs) + g.Expect(err).To(Not(HaveOccurred())) + }) }) t.Run("Reconcile failure", func(t *testing.T) { expectedErr := errors.New("failed to get resource") diff --git a/controllers/awsmachine_controller.go b/controllers/awsmachine_controller.go index 14bb9387a1..e468e4c2d7 100644 --- a/controllers/awsmachine_controller.go +++ b/controllers/awsmachine_controller.go @@ -409,6 +409,15 @@ func (r *AWSMachineReconciler) reconcileDelete(machineScope *scope.MachineScope, conditions.MarkFalse(machineScope.AWSMachine, infrav1.SecurityGroupsReadyCondition, clusterv1.DeletedReason, clusterv1.ConditionSeverityInfo, "") } + // Release an Elastic IP when the machine has public IP Address (EIP) with a cluster-wide config + // to consume from BYO IPv4 Pool. + if machineScope.GetElasticIPPool() != nil { + if err := ec2Service.ReleaseElasticIP(instance.ID); err != nil { + machineScope.Error(err, "failed to release elastic IP address") + return ctrl.Result{}, err + } + } + machineScope.Info("EC2 instance successfully terminated", "instance-id", instance.ID) r.Recorder.Eventf(machineScope.AWSMachine, corev1.EventTypeNormal, "SuccessfulTerminate", "Terminated instance %q", instance.ID) @@ -522,6 +531,21 @@ func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope * return ctrl.Result{}, err } } + + // BYO Public IPv4 Pool feature: allocates and associates an EIP to machine when PublicIP and + // cluster-wide Public IPv4 Pool configuration are set. The EIP must be associated after the instance + // is created and transictioned from Pending state. + // In the regular flow, if the instance have already a public IPv4 address (EIP) associated it will + // be released when a new is assigned, the createInstance() prevents that behavior by enforcing + // to not launch an instance with EIP, allowing ReconcileElasticIPFromPublicPool assigning + // a BYOIP without duplication. + if pool := machineScope.GetElasticIPPool(); pool != nil { + if err := ec2svc.ReconcileElasticIPFromPublicPool(pool, instance); err != nil { + machineScope.Error(err, "failed to associate elastic IP address") + return ctrl.Result{}, err + } + } + if feature.Gates.Enabled(feature.EventBridgeInstanceState) { instancestateSvc := instancestate.NewService(ec2Scope) if err := instancestateSvc.AddInstanceToEventPattern(instance.ID); err != nil { diff --git a/controllers/awsmachine_controller_test.go b/controllers/awsmachine_controller_test.go index c6faf254d4..b96047a9e9 100644 --- a/controllers/awsmachine_controller_test.go +++ b/controllers/awsmachine_controller_test.go @@ -154,6 +154,8 @@ func TestAWSMachineReconcilerIntegrationTests(t *testing.T) { return elbSvc } + ec2Mock.EXPECT().AssociateAddressWithContext(context.TODO(), gomock.Any()).MaxTimes(1) + reconciler.secretsManagerServiceFactory = func(clusterScope cloud.ClusterScoper) services.SecretInterface { return secretMock } @@ -330,6 +332,8 @@ func TestAWSMachineReconcilerIntegrationTests(t *testing.T) { return secretMock } + ec2Mock.EXPECT().AssociateAddressWithContext(context.TODO(), gomock.Any()).MaxTimes(1) + _, err = reconciler.reconcileNormal(ctx, ms, cs, cs, cs, cs) g.Expect(err).Should(HaveOccurred()) expectConditions(g, ms.AWSMachine, []conditionAssertion{{infrav1.InstanceReadyCondition, corev1.ConditionTrue, "", ""}}) diff --git a/controllers/awsmachine_controller_unit_test.go b/controllers/awsmachine_controller_unit_test.go index 68cd68981a..78058f3cec 100644 --- a/controllers/awsmachine_controller_unit_test.go +++ b/controllers/awsmachine_controller_unit_test.go @@ -995,6 +995,40 @@ func TestAWSMachineReconciler(t *testing.T) { }) }) }) + t.Run("when BYOIP is set", func(t *testing.T) { + var instance *infrav1.Instance + secretPrefix := "test/secret" + + t.Run("should succeed", func(t *testing.T) { + g := NewWithT(t) + awsMachine := getAWSMachine() + setup(t, g, awsMachine) + defer teardown(t, g) + + instance = &infrav1.Instance{ + ID: "myMachine", + State: infrav1.InstanceStatePending, + } + + ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(nil, nil).AnyTimes() + secretSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return(secretPrefix, int32(1), nil).Times(1) + ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any(), gomock.Any()).Return(instance, nil).AnyTimes() + secretSvc.EXPECT().UserData(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) + ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1) + ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1) + ec2Svc.EXPECT().GetAdditionalSecurityGroupsIDs(gomock.Any()).Return(nil, nil).Times(1) + + ms.AWSMachine.Spec.PublicIP = aws.Bool(false) + ms.AWSMachine.Spec.ElasticIPPool = &infrav1.ElasticIPPool{ + PublicIpv4Pool: aws.String("ipv4pool-ec2-0123456789abcdef0"), + PublicIpv4PoolFallBackOrder: ptr.To(infrav1.PublicIpv4PoolFallbackOrderAmazonPool), + } + ec2Svc.EXPECT().ReconcileElasticIPFromPublicPool(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + _, err := reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs) + g.Expect(err).To(BeNil()) + }) + }) }) t.Run("Secrets management lifecycle", func(t *testing.T) { @@ -2722,6 +2756,7 @@ func TestAWSMachineReconcilerReconcileDefaultsToLoadBalancerTypeClassic(t *testi Attribute: aws.String("groupSet"), })).Return(&ec2.DescribeNetworkInterfaceAttributeOutput{Groups: []*ec2.GroupIdentifier{{GroupId: aws.String("3")}}}, nil).MaxTimes(1) ec2Mock.EXPECT().ModifyNetworkInterfaceAttributeWithContext(context.TODO(), gomock.Any()).AnyTimes() + ec2Mock.EXPECT().AssociateAddressWithContext(context.TODO(), gomock.Any()).MaxTimes(1) _, err = reconciler.Reconcile(ctx, ctrl.Request{ NamespacedName: client.ObjectKey{ diff --git a/controlplane/eks/controllers/awsmanagedcontrolplane_controller_test.go b/controlplane/eks/controllers/awsmanagedcontrolplane_controller_test.go index ca7b1a2ec9..8dc78e544b 100644 --- a/controlplane/eks/controllers/awsmanagedcontrolplane_controller_test.go +++ b/controlplane/eks/controllers/awsmanagedcontrolplane_controller_test.go @@ -502,7 +502,7 @@ func mockedCallsForMissingEverything(ec2Rec *mocks.MockEC2APIMockRecorder, subne }, { Name: aws.String("tag:sigs.k8s.io/cluster-api-provider-aws/role"), - Values: aws.StringSlice([]string{"apiserver"}), + Values: aws.StringSlice([]string{"common"}), }, }, })).Return(&ec2.DescribeAddressesOutput{ @@ -525,7 +525,7 @@ func mockedCallsForMissingEverything(ec2Rec *mocks.MockEC2APIMockRecorder, subne Tags: []*ec2.Tag{ { Key: aws.String("Name"), - Value: aws.String("test-cluster-eip-apiserver"), + Value: aws.String("test-cluster-eip-common"), }, { Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), @@ -533,7 +533,7 @@ func mockedCallsForMissingEverything(ec2Rec *mocks.MockEC2APIMockRecorder, subne }, { Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), - Value: aws.String("apiserver"), + Value: aws.String("common"), }, }, }, diff --git a/docs/book/src/topics/bring-your-own-aws-infrastructure.md b/docs/book/src/topics/bring-your-own-aws-infrastructure.md index bd157fa28f..a5cd878a9f 100644 --- a/docs/book/src/topics/bring-your-own-aws-infrastructure.md +++ b/docs/book/src/topics/bring-your-own-aws-infrastructure.md @@ -274,3 +274,69 @@ The external system must provide all required fields within the spec of the AWSC Once the user has created externally managed AWSCluster, it is not allowed to convert it to CAPA managed cluster. However, converting from managed to externally managed is allowed. User should only use this feature if their cluster infrastructure lifecycle management has constraints that the reference implementation does not support. See [user stories](https://github.com/kubernetes-sigs/cluster-api/blob/10d89ceca938e4d3d94a1d1c2b60515bcdf39829/docs/proposals/20210203-externally-managed-cluster-infrastructure.md#user-stories) for more details. + + +## Bring your own (BYO) Public IPv4 addresses + +Cluster API also provides a mechanism to allocate Elastic IP from the existing Public IPv4 Pool that you brought to AWS[1]. + +Bringing your own Public IPv4 Pool (BYOIPv4) can be used as an alternative to buying Public IPs from AWS, also considering the changes in charging for this since February 2024[2]. + +Supported resources to BYO Public IPv4 Pool (`BYO Public IPv4`): +- NAT Gateways +- Network Load Balancer for API server +- Machines + +Use `BYO Public IPv4` when you have brought to AWS custom IPv4 CIDR blocks and want the cluster to automatically use IPs from the custom pool instead of Amazon-provided pools. + +### Prerequisites and limitations for BYO Public IPv4 Pool + +- BYOIPv4 is limited to AWS to selected regions. See more in [AWS Documentation for Regional availability](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-byoip.html#byoip-reg-avail) +- The IPv4 address must be provisioned and advertised to the AWS account before the cluster is installed +- The public IPv4 addresses is limited to the network border group that the CIDR block have been advertised[3][4], and the `NetworkSpec.ElasticIpPool.PublicIpv4Pool` must be the same of the cluster will be installed. +- Only NAT Gateways and the Network Load Balancer for API server will consume from the IPv4 pool defined in the network scope. +- The public IPv4 pool must be assigned to each machine to consume public IPv4 from a custom IPv4 pool. + +### Steps to set BYO Public IPv4 Pool to core infrastructure + +Currently, CAPA supports BYO Public IPv4 to core components NAT Gateways and Network Load Balancer for the internet-facing API server. + +To specify a Public IPv4 Pool for core components you must set the `spec.elasticIpPool` as follows: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSCluster +metadata: + name: aws-cluster-localzone +spec: + region: us-east-1 + networkSpec: + vpc: + elasticIpPool: + publicIpv4Pool: ipv4pool-ec2-0123456789abcdef0 + publicIpv4PoolFallbackOrder: amazon-pool +``` + +Then all the Elastic IPs will be created by consuming from the pool `ipv4pool-ec2-0123456789abcdef0`. + +### Steps to BYO Public IPv4 Pool to machines + +To create a machine consuming from a custom Public IPv4 Pool you must set the pool ID to the AWSMachine spec, then set the `PublicIP` to `true`: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachine +metadata: + name: byoip-s55p4-bootstrap +spec: + # placeholder for AWSMachine spec + elasticIpPool: + publicIpv4Pool: ipv4pool-ec2-0123456789abcdef0 + publicIpv4PoolFallbackOrder: amazon-pool + publicIP: true +``` + +[1] https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-byoip.html +[2] https://aws.amazon.com/blogs/aws/new-aws-public-ipv4-address-charge-public-ip-insights/ +[3] https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-byoip.html#byoip-onboard +[4] https://docs.aws.amazon.com/cli/latest/reference/ec2/advertise-byoip-cidr.html diff --git a/pkg/cloud/scope/machine.go b/pkg/cloud/scope/machine.go index f547f284cb..331c4c31e2 100644 --- a/pkg/cloud/scope/machine.go +++ b/pkg/cloud/scope/machine.go @@ -400,3 +400,11 @@ func (m *MachineScope) SetInterruptible() { m.AWSMachine.Status.Interruptible = true } } + +// GetElasticIPPool returns the Elastic IP Pool for an machine, when exists. +func (m *MachineScope) GetElasticIPPool() *infrav1.ElasticIPPool { + if m.AWSMachine == nil { + return nil + } + return m.AWSMachine.Spec.ElasticIPPool +} diff --git a/pkg/cloud/services/ec2/eip.go b/pkg/cloud/services/ec2/eip.go new file mode 100644 index 0000000000..77484e201c --- /dev/null +++ b/pkg/cloud/services/ec2/eip.go @@ -0,0 +1,54 @@ +package ec2 + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/record" +) + +func getElasticIPRoleName(instanceID string) string { + return fmt.Sprintf("ec2-%s", instanceID) +} + +// ReconcileElasticIPFromPublicPool reconciles the elastic IP from a custom Public IPv4 Pool. +func (s *Service) ReconcileElasticIPFromPublicPool(pool *infrav1.ElasticIPPool, instance *infrav1.Instance) error { + // TODO: check if the instance is in the state allowing EIP association. + // Expected instance states: pending or running + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-lifecycle.html + if err := s.getAndAssociateAddressesToInstance(pool, getElasticIPRoleName(instance.ID), instance.ID); err != nil { + return fmt.Errorf("failed to reconcile Elastic IP: %w", err) + } + return nil +} + +// ReleaseElasticIP releases a specific elastic IP based on the instance role. +func (s *Service) ReleaseElasticIP(instanceID string) error { + return s.netService.ReleaseAddressByRole(getElasticIPRoleName(instanceID)) +} + +// getAndAssociateAddressesToInstance find or create an EIP from an instance and role. +func (s *Service) getAndAssociateAddressesToInstance(pool *infrav1.ElasticIPPool, role string, instance string) (err error) { + eips, err := s.netService.GetOrAllocateAddresses(pool, 1, role) + if err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedAllocateEIP", "Failed to get Elastic IP for %q: %v", role, err) + return err + } + if len(eips) != 1 { + record.Warnf(s.scope.InfraCluster(), "FailedAllocateEIP", "Failed to allocate Elastic IP for %q: %v", role, err) + return fmt.Errorf("unexpected number of Elastic IP to instance %q, got %d: %w", instance, len(eips), err) + } + _, err = s.EC2Client.AssociateAddressWithContext(context.TODO(), &ec2.AssociateAddressInput{ + InstanceId: aws.String(instance), + AllocationId: aws.String(eips[0]), + }) + if err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedAssociateEIP", "Failed to associate Elastic IP for %q: %v", role, err) + return fmt.Errorf("failed to associate Elastic IP %q to instance %q: %w", eips[0], instance, err) + } + return nil +} diff --git a/pkg/cloud/services/ec2/instances.go b/pkg/cloud/services/ec2/instances.go index a3efbea726..b6975c5309 100644 --- a/pkg/cloud/services/ec2/instances.go +++ b/pkg/cloud/services/ec2/instances.go @@ -182,19 +182,14 @@ func (s *Service) CreateInstance(scope *scope.MachineScope, userData []byte, use } input.SubnetID = subnetID - if ptr.Deref(scope.AWSMachine.Spec.PublicIP, false) { - subnets, err := s.getFilteredSubnets(&ec2.Filter{ - Name: aws.String("subnet-id"), - Values: aws.StringSlice([]string{subnetID}), - }) - if err != nil { - return nil, fmt.Errorf("could not query if subnet has MapPublicIpOnLaunch set: %w", err) - } - if len(subnets) == 0 { - return nil, fmt.Errorf("expected to find subnet %q", subnetID) - } - // If the subnet does not assign public IPs, set that option in the instance's network interface - input.PublicIPOnLaunch = ptr.To(!aws.BoolValue(subnets[0].MapPublicIpOnLaunch)) + // Preserve user-defined PublicIp option. + input.PublicIPOnLaunch = scope.AWSMachine.Spec.PublicIP + + // Public address from BYO Public IPv4 Pools need to be associated after launch (main machine + // reconciliate loop) preventing duplicated public IP. The map on launch is explicitly + // disabled in instances with PublicIP defined to true. + if scope.AWSMachine.Spec.ElasticIPPool != nil && scope.AWSMachine.Spec.ElasticIPPool.PublicIpv4Pool != nil { + input.PublicIPOnLaunch = ptr.To(false) } if !scope.IsControlPlaneExternallyManaged() && !scope.IsExternallyManaged() && !scope.IsEKSManaged() && s.scope.Network().APIServerELB.DNSName == "" { diff --git a/pkg/cloud/services/ec2/instances_test.go b/pkg/cloud/services/ec2/instances_test.go index e87ac6c9c3..5fc267036b 100644 --- a/pkg/cloud/services/ec2/instances_test.go +++ b/pkg/cloud/services/ec2/instances_test.go @@ -2129,19 +2129,6 @@ func TestCreateInstance(t *testing.T) { MapPublicIpOnLaunch: aws.Bool(true), }}, }, nil) - m. - DescribeSubnetsWithContext(context.TODO(), &ec2.DescribeSubnetsInput{ - Filters: []*ec2.Filter{ - {Name: aws.String("subnet-id"), Values: aws.StringSlice([]string{"public-subnet-1"})}, - }, - }). - Return(&ec2.DescribeSubnetsOutput{ - Subnets: []*ec2.Subnet{{ - SubnetId: aws.String("public-subnet-1"), - AvailabilityZone: aws.String("us-east-1b"), - MapPublicIpOnLaunch: aws.Bool(true), - }}, - }, nil) m. DescribeInstanceTypesWithContext(context.TODO(), gomock.Eq(&ec2.DescribeInstanceTypesInput{ InstanceTypes: []*string{ @@ -2271,19 +2258,6 @@ func TestCreateInstance(t *testing.T) { MapPublicIpOnLaunch: aws.Bool(false), }}, }, nil) - m. - DescribeSubnetsWithContext(context.TODO(), &ec2.DescribeSubnetsInput{ - Filters: []*ec2.Filter{ - {Name: aws.String("subnet-id"), Values: aws.StringSlice([]string{"public-subnet-1"})}, - }, - }). - Return(&ec2.DescribeSubnetsOutput{ - Subnets: []*ec2.Subnet{{ - SubnetId: aws.String("public-subnet-1"), - AvailabilityZone: aws.String("us-east-1b"), - MapPublicIpOnLaunch: aws.Bool(false), - }}, - }, nil) m. DescribeInstanceTypesWithContext(context.TODO(), gomock.Eq(&ec2.DescribeInstanceTypesInput{ InstanceTypes: []*string{ @@ -2550,18 +2524,6 @@ func TestCreateInstance(t *testing.T) { MapPublicIpOnLaunch: aws.Bool(true), }}, }, nil) - m. - DescribeSubnetsWithContext(context.TODO(), &ec2.DescribeSubnetsInput{ - Filters: []*ec2.Filter{ - {Name: aws.String("subnet-id"), Values: aws.StringSlice([]string{"public-subnet-1"})}, - }, - }). - Return(&ec2.DescribeSubnetsOutput{ - Subnets: []*ec2.Subnet{{ - SubnetId: aws.String("public-subnet-1"), - MapPublicIpOnLaunch: aws.Bool(true), - }}, - }, nil) m. RunInstancesWithContext(context.TODO(), gomock.Any()). Return(&ec2.Reservation{ @@ -2699,18 +2661,6 @@ func TestCreateInstance(t *testing.T) { MapPublicIpOnLaunch: aws.Bool(false), }}, }, nil) - m. - DescribeSubnetsWithContext(context.TODO(), &ec2.DescribeSubnetsInput{ - Filters: []*ec2.Filter{ - {Name: aws.String("subnet-id"), Values: aws.StringSlice([]string{"public-subnet-1"})}, - }, - }). - Return(&ec2.DescribeSubnetsOutput{ - Subnets: []*ec2.Subnet{{ - SubnetId: aws.String("public-subnet-1"), - MapPublicIpOnLaunch: aws.Bool(false), - }}, - }, nil) m. RunInstancesWithContext(context.TODO(), gomock.Any()). Do(func(_ context.Context, in *ec2.RunInstancesInput, _ ...request.Option) { @@ -2843,18 +2793,6 @@ func TestCreateInstance(t *testing.T) { }, }, }, nil) - m. - DescribeSubnetsWithContext(context.TODO(), &ec2.DescribeSubnetsInput{ - Filters: []*ec2.Filter{ - {Name: aws.String("subnet-id"), Values: aws.StringSlice([]string{"public-subnet-1"})}, - }, - }). - Return(&ec2.DescribeSubnetsOutput{ - Subnets: []*ec2.Subnet{{ - SubnetId: aws.String("public-subnet-1"), - MapPublicIpOnLaunch: aws.Bool(true), - }}, - }, nil) m. RunInstancesWithContext(context.TODO(), gomock.Any()). Return(&ec2.Reservation{ @@ -2973,18 +2911,6 @@ func TestCreateInstance(t *testing.T) { }, }, }, nil) - m. - DescribeSubnetsWithContext(context.TODO(), &ec2.DescribeSubnetsInput{ - Filters: []*ec2.Filter{ - {Name: aws.String("subnet-id"), Values: aws.StringSlice([]string{"public-subnet-1"})}, - }, - }). - Return(&ec2.DescribeSubnetsOutput{ - Subnets: []*ec2.Subnet{{ - SubnetId: aws.String("public-subnet-1"), - MapPublicIpOnLaunch: aws.Bool(false), - }}, - }, nil) m. RunInstancesWithContext(context.TODO(), gomock.Any()). Do(func(_ context.Context, in *ec2.RunInstancesInput, _ ...request.Option) { diff --git a/pkg/cloud/services/ec2/service.go b/pkg/cloud/services/ec2/service.go index b085ee86c8..03e08b1203 100644 --- a/pkg/cloud/services/ec2/service.go +++ b/pkg/cloud/services/ec2/service.go @@ -22,14 +22,16 @@ import ( "github.com/aws/aws-sdk-go/service/ssm/ssmiface" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network" ) // Service holds a collection of interfaces. // The interfaces are broken down like this to group functions together. // One alternative is to have a large list of functions from the ec2 client. type Service struct { - scope scope.EC2Scope - EC2Client ec2iface.EC2API + scope scope.EC2Scope + EC2Client ec2iface.EC2API + netService *network.Service // SSMClient is used to look up the official EKS AMI ID SSMClient ssmiface.SSMAPI @@ -38,8 +40,9 @@ type Service struct { // NewService returns a new service given the ec2 api client. func NewService(clusterScope scope.EC2Scope) *Service { return &Service{ - scope: clusterScope, - EC2Client: scope.NewEC2Client(clusterScope, clusterScope, clusterScope, clusterScope.InfraCluster()), - SSMClient: scope.NewSSMClient(clusterScope, clusterScope, clusterScope, clusterScope.InfraCluster()), + scope: clusterScope, + EC2Client: scope.NewEC2Client(clusterScope, clusterScope, clusterScope, clusterScope.InfraCluster()), + SSMClient: scope.NewSSMClient(clusterScope, clusterScope, clusterScope, clusterScope.InfraCluster()), + netService: network.NewService(clusterScope.(scope.NetworkScope)), } } diff --git a/pkg/cloud/services/elb/eip.go b/pkg/cloud/services/elb/eip.go new file mode 100644 index 0000000000..27c9ebee70 --- /dev/null +++ b/pkg/cloud/services/elb/eip.go @@ -0,0 +1,55 @@ +package elb + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elbv2" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" +) + +func getElasticIPRoleName() string { + return fmt.Sprintf("lb-%s", infrav1.APIServerRoleTagValue) +} + +// allocatePublicIpv4AddressFromByoIPPool claims for Elastic IPs from an user-defined public IPv4 pool, +// allocating it to the NetworkMapping structure from an Network Load Balancer. +func (s *Service) allocatePublicIpv4AddressFromByoIPPool(input *elbv2.CreateLoadBalancerInput) error { + // Custom Public IPv4 Pool isn't set. + if s.scope.VPC().GetPublicIpv4Pool() == nil { + return nil + } + + // Only NLB is supported + if input.Type == nil { + return fmt.Errorf("PublicIpv4Pool is supported only when the Load Balancer type is %q", elbv2.LoadBalancerTypeEnumNetwork) + } + if *input.Type != string(elbv2.LoadBalancerTypeEnumNetwork) { + return fmt.Errorf("PublicIpv4Pool is not supported with Load Balancer type %s. Use Network Load Balancer instead", *input.Type) + } + + // Custom SubnetMappings should not be defined or overridden by user-defined mapping. + if len(input.SubnetMappings) > 0 { + return fmt.Errorf("PublicIpv4Pool is mutually exclusive with SubnetMappings") + } + + eips, err := s.netService.GetOrAllocateAddresses(s.scope.VPC().GetElasticIPPool(), len(input.Subnets), getElasticIPRoleName()) + if err != nil { + return fmt.Errorf("failed to allocate address from Public IPv4 Pool %q to role %s: %w", *s.scope.VPC().GetPublicIpv4Pool(), getElasticIPRoleName(), err) + } + if len(eips) != len(input.Subnets) { + return fmt.Errorf("number of allocated EIP addresses (%d) from pool %q must match with the subnet count (%d)", len(eips), *s.scope.VPC().GetPublicIpv4Pool(), len(input.Subnets)) + } + for cnt, sb := range input.Subnets { + input.SubnetMappings = append(input.SubnetMappings, &elbv2.SubnetMapping{ + SubnetId: aws.String(*sb), + AllocationId: aws.String(eips[cnt]), + }) + } + // Subnets and SubnetMappings are mutual exclusive. Cleaning Subnets when BYO IP is defined, + // and SubnetMappings are mounted. + input.Subnets = []*string{} + + return nil +} diff --git a/pkg/cloud/services/elb/loadbalancer.go b/pkg/cloud/services/elb/loadbalancer.go index 234123b4d5..f02a084ada 100644 --- a/pkg/cloud/services/elb/loadbalancer.go +++ b/pkg/cloud/services/elb/loadbalancer.go @@ -379,6 +379,22 @@ func (s *Service) createLB(spec *infrav1.LoadBalancer, lbSpec *infrav1.AWSLoadBa input.IpAddressType = aws.String("dualstack") } + // Allocate custom addresses (Elastic IP) to internet-facing Load Balancers, when defined. + // Custom, or BYO, Public IPv4 Pool need to be created prior install, and the Pool ID must be + // set in the VpcSpec.ElasticIPPool.PublicIPv4Pool to allow Elastic IP be consumed from + // public ip address of user-provided CIDR blocks. + if spec.Scheme == infrav1.ELBSchemeInternetFacing { + if err := s.allocatePublicIpv4AddressFromByoIPPool(input); err != nil { + return nil, fmt.Errorf("failed to allocate addresses to load balancer: %w", err) + } + } + + // Subnets and SubnetMappings are mutually exclusive. SubnetMappings is set by users or when + // BYO Public IPv4 Pool is set. + if len(input.SubnetMappings) == 0 { + input.Subnets = aws.StringSlice(spec.SubnetIDs) + } + out, err := s.ELBV2Client.CreateLoadBalancer(input) if err != nil { return nil, errors.Wrapf(err, "failed to create load balancer: %v", spec) diff --git a/pkg/cloud/services/elb/service.go b/pkg/cloud/services/elb/service.go index c0717c6f25..709329001b 100644 --- a/pkg/cloud/services/elb/service.go +++ b/pkg/cloud/services/elb/service.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network" ) // Service holds a collection of interfaces. @@ -35,6 +36,7 @@ type Service struct { ELBClient elbiface.ELBAPI ELBV2Client elbv2iface.ELBV2API ResourceTaggingClient resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI + netService *network.Service } // NewService returns a new service given the api clients. @@ -45,5 +47,6 @@ func NewService(elbScope scope.ELBScope) *Service { ELBClient: scope.NewELBClient(elbScope, elbScope, elbScope, elbScope.InfraCluster()), ELBV2Client: scope.NewELBv2Client(elbScope, elbScope, elbScope, elbScope.InfraCluster()), ResourceTaggingClient: scope.NewResourgeTaggingClient(elbScope, elbScope, elbScope, elbScope.InfraCluster()), + netService: network.NewService(elbScope.(scope.NetworkScope)), } } diff --git a/pkg/cloud/services/interfaces.go b/pkg/cloud/services/interfaces.go index 46a2c7aecf..f993d9bd84 100644 --- a/pkg/cloud/services/interfaces.go +++ b/pkg/cloud/services/interfaces.go @@ -81,6 +81,11 @@ type EC2Interface interface { LaunchTemplateNeedsUpdate(scope scope.LaunchTemplateScope, incoming *expinfrav1.AWSLaunchTemplate, existing *expinfrav1.AWSLaunchTemplate) (bool, error) DeleteBastion() error ReconcileBastion() error + // ReconcileElasticIPFromPublicPool reconciles the elastic IP from a custom Public IPv4 Pool. + ReconcileElasticIPFromPublicPool(pool *infrav1.ElasticIPPool, instance *infrav1.Instance) error + + // ReleaseElasticIP reconciles the elastic IP from a custom Public IPv4 Pool. + ReleaseElasticIP(instanceID string) error } // MachinePoolReconcileInterface encapsulates high-level reconciliation functions regarding EC2 reconciliation. It is diff --git a/pkg/cloud/services/mock_services/ec2_interface_mock.go b/pkg/cloud/services/mock_services/ec2_interface_mock.go index 922d5f3360..02c3e09c47 100644 --- a/pkg/cloud/services/mock_services/ec2_interface_mock.go +++ b/pkg/cloud/services/mock_services/ec2_interface_mock.go @@ -333,6 +333,34 @@ func (mr *MockEC2InterfaceMockRecorder) ReconcileBastion() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileBastion", reflect.TypeOf((*MockEC2Interface)(nil).ReconcileBastion)) } +// ReconcileElasticIPFromPublicPool mocks base method. +func (m *MockEC2Interface) ReconcileElasticIPFromPublicPool(arg0 *v1beta2.ElasticIPPool, arg1 *v1beta2.Instance) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReconcileElasticIPFromPublicPool", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReconcileElasticIPFromPublicPool indicates an expected call of ReconcileElasticIPFromPublicPool. +func (mr *MockEC2InterfaceMockRecorder) ReconcileElasticIPFromPublicPool(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileElasticIPFromPublicPool", reflect.TypeOf((*MockEC2Interface)(nil).ReconcileElasticIPFromPublicPool), arg0, arg1) +} + +// ReleaseElasticIP mocks base method. +func (m *MockEC2Interface) ReleaseElasticIP(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReleaseElasticIP", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReleaseElasticIP indicates an expected call of ReleaseElasticIP. +func (mr *MockEC2InterfaceMockRecorder) ReleaseElasticIP(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReleaseElasticIP", reflect.TypeOf((*MockEC2Interface)(nil).ReleaseElasticIP), arg0) +} + // TerminateInstance mocks base method. func (m *MockEC2Interface) TerminateInstance(arg0 string) error { m.ctrl.T.Helper() diff --git a/pkg/cloud/services/network/eips.go b/pkg/cloud/services/network/eips.go index 666f96652e..f301650797 100644 --- a/pkg/cloud/services/network/eips.go +++ b/pkg/cloud/services/network/eips.go @@ -32,41 +32,53 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/record" ) -func (s *Service) getOrAllocateAddresses(num int, role string) (eips []string, err error) { +func (s *Service) getOrAllocateAddresses(num int, role string, pool *infrav1.ElasticIPPool) (eips []string, err error) { out, err := s.describeAddresses(role) if err != nil { record.Eventf(s.scope.InfraCluster(), "FailedDescribeAddresses", "Failed to query addresses for role %q: %v", role, err) return nil, errors.Wrap(err, "failed to query addresses") } + // Reuse existing unallocated addreses with the same role. for _, address := range out.Addresses { if address.AssociationId == nil { eips = append(eips, aws.StringValue(address.AllocationId)) } } + // allocate addresses when needed. + tagSpecifications := tags.BuildParamsToTagSpecification(ec2.ResourceTypeElasticIp, s.getEIPTagParams(role)) for len(eips) < num { - ip, err := s.allocateAddress(role) - if err != nil { + allocInput := &ec2.AllocateAddressInput{ + Domain: aws.String("vpc"), + TagSpecifications: []*ec2.TagSpecification{ + tagSpecifications, + }, + } + + // Set EIP to consume from BYO Public IPv4 pools when defined in NetworkSpec with preflight checks. + // The checks makes sure there is free IPs available in the pool before allocating it. + // The check also validate the fallback strategy to consume from Amazon pool when the + // pool is exchausted. + if err := s.setByoPublicIpv4(pool, allocInput); err != nil { return nil, err } + + ip, err := s.allocateAddress(allocInput) + if err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedAllocateAddress", "Failed to allocate Elastic IP for %q: %v", role, err) + return nil, fmt.Errorf("failed to allocate Elastic IP for %q: %w", role, err) + } eips = append(eips, ip) } return eips, nil } -func (s *Service) allocateAddress(role string) (string, error) { - tagSpecifications := tags.BuildParamsToTagSpecification(ec2.ResourceTypeElasticIp, s.getEIPTagParams(role)) - out, err := s.EC2Client.AllocateAddressWithContext(context.TODO(), &ec2.AllocateAddressInput{ - Domain: aws.String("vpc"), - TagSpecifications: []*ec2.TagSpecification{ - tagSpecifications, - }, - }) +func (s *Service) allocateAddress(alloc *ec2.AllocateAddressInput) (string, error) { + out, err := s.EC2Client.AllocateAddressWithContext(context.TODO(), alloc) if err != nil { - record.Warnf(s.scope.InfraCluster(), "FailedAllocateEIP", "Failed to allocate Elastic IP for %q: %v", role, err) - return "", errors.Wrap(err, "failed to allocate Elastic IP") + return "", err } return aws.StringValue(out.AllocationId), nil @@ -103,9 +115,42 @@ func (s *Service) disassociateAddress(ip *ec2.Address) error { return nil } -func (s *Service) releaseAddresses() error { +// releaseAddress releases an given EIP address back to the pool. +func (s *Service) releaseAddress(ip *ec2.Address) error { + if ip.AssociationId != nil { + if _, err := s.EC2Client.DisassociateAddressWithContext(context.TODO(), &ec2.DisassociateAddressInput{ + AssociationId: ip.AssociationId, + }); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedDisassociateEIP", "Failed to disassociate Elastic IP %q: %v", *ip.AllocationId, err) + return errors.Errorf("failed to disassociate Elastic IP %q with allocation ID %q: Still associated with association ID %q", *ip.PublicIp, *ip.AllocationId, *ip.AssociationId) + } + } + + if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { + _, err := s.EC2Client.ReleaseAddressWithContext(context.TODO(), &ec2.ReleaseAddressInput{AllocationId: ip.AllocationId}) + if err != nil { + if ip.AssociationId != nil { + if s.disassociateAddress(ip) != nil { + return false, err + } + } + return false, err + } + return true, nil + }, awserrors.AuthFailure, awserrors.InUseIPAddress); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedReleaseEIP", "Failed to disassociate Elastic IP %q: %v", *ip.AllocationId, err) + return errors.Wrapf(err, "failed to release ElasticIP %q", *ip.AllocationId) + } + + s.scope.Info("released ElasticIP", "eip", *ip.PublicIp, "allocation-id", *ip.AllocationId) + return nil +} + +// releaseAddressesWithFilter discovery address to be released based in filters, returning no error, +// when all addresses have been released. +func (s *Service) releaseAddressesWithFilter(filters []*ec2.Filter) error { out, err := s.EC2Client.DescribeAddressesWithContext(context.TODO(), &ec2.DescribeAddressesInput{ - Filters: []*ec2.Filter{filter.EC2.Cluster(s.scope.Name())}, + Filters: filters, }) if err != nil { return errors.Wrapf(err, "failed to describe elastic IPs %q", err) @@ -114,37 +159,21 @@ func (s *Service) releaseAddresses() error { return nil } for i := range out.Addresses { - ip := out.Addresses[i] - if ip.AssociationId != nil { - if _, err := s.EC2Client.DisassociateAddressWithContext(context.TODO(), &ec2.DisassociateAddressInput{ - AssociationId: ip.AssociationId, - }); err != nil { - record.Warnf(s.scope.InfraCluster(), "FailedDisassociateEIP", "Failed to disassociate Elastic IP %q: %v", *ip.AllocationId, err) - return errors.Errorf("failed to disassociate Elastic IP %q with allocation ID %q: Still associated with association ID %q", *ip.PublicIp, *ip.AllocationId, *ip.AssociationId) - } - } - - if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { - _, err := s.EC2Client.ReleaseAddressWithContext(context.TODO(), &ec2.ReleaseAddressInput{AllocationId: ip.AllocationId}) - if err != nil { - if ip.AssociationId != nil { - if s.disassociateAddress(ip) != nil { - return false, err - } - } - return false, err - } - return true, nil - }, awserrors.AuthFailure, awserrors.InUseIPAddress); err != nil { - record.Warnf(s.scope.InfraCluster(), "FailedReleaseEIP", "Failed to disassociate Elastic IP %q: %v", *ip.AllocationId, err) - return errors.Wrapf(err, "failed to release ElasticIP %q", *ip.AllocationId) + if err := s.releaseAddress(out.Addresses[i]); err != nil { + return err } - - s.scope.Info("released ElasticIP", "eip", *ip.PublicIp, "allocation-id", *ip.AllocationId) } return nil } +// releaseAddresses is default cluster release flow, discoverying and releasing all +// addresses associated and owned by the cluster tag. +func (s *Service) releaseAddresses() error { + filters := []*ec2.Filter{filter.EC2.Cluster(s.scope.Name())} + filters = append(filters, filter.EC2.ClusterOwned(s.scope.Name())) + return s.releaseAddressesWithFilter(filters) +} + func (s *Service) getEIPTagParams(role string) infrav1.BuildParams { name := fmt.Sprintf("%s-eip-%s", s.scope.Name(), role) @@ -156,3 +185,75 @@ func (s *Service) getEIPTagParams(role string) infrav1.BuildParams { Additional: s.scope.AdditionalTags(), } } + +// GetOrAllocateAddresses exports the interface to allocate an address from external services. +func (s *Service) GetOrAllocateAddresses(pool *infrav1.ElasticIPPool, num int, role string) (eips []string, err error) { + return s.getOrAllocateAddresses(num, role, pool) +} + +// ReleaseAddressByRole releases EIP addresses filtering by tag CAPA provider role. +func (s *Service) ReleaseAddressByRole(role string) error { + clusterFilter := []*ec2.Filter{filter.EC2.Cluster(s.scope.Name())} + clusterFilter = append(clusterFilter, filter.EC2.ProviderRole(role)) + + return s.releaseAddressesWithFilter(clusterFilter) +} + +// setByoPublicIpv4 check if the config has Public IPv4 Pool defined, then +// check if there are IPs available to consume from allocation, otherwise +// fallback to Amazon pool when explicty failure isn't defined. +func (s *Service) setByoPublicIpv4(pool *infrav1.ElasticIPPool, alloc *ec2.AllocateAddressInput) error { + if pool == nil { + return nil + } + // check if pool has free IP. + ok, err := s.publicIpv4PoolHasAtLeastNFreeIPs(pool, 1) + if err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedAllocateEIP", "Failed to allocate Elastic IP from Public IPv4 pool %q: %w", *pool.PublicIpv4Pool, err) + return fmt.Errorf("failed to update Elastic IP: %w", err) + } + + // use the custom public ipv4 pool to the Elastic IP allocation. + if ok { + alloc.PublicIpv4Pool = pool.PublicIpv4Pool + return nil + } + + // default, don't change allocation config, use Amazon pool. + return nil +} + +// publicIpv4PoolHasAtLeastNFreeIPs check if there are N IPs address available in a Public IPv4 Pool. +func (s *Service) publicIpv4PoolHasAtLeastNFreeIPs(pool *infrav1.ElasticIPPool, want int64) (bool, error) { + if pool == nil { + return true, nil + } + if pool.PublicIpv4Pool == nil { + return true, nil + } + publicIpv4Pool := pool.PublicIpv4Pool + pools, err := s.EC2Client.DescribePublicIpv4Pools(&ec2.DescribePublicIpv4PoolsInput{ + PoolIds: []*string{publicIpv4Pool}, + }) + if err != nil { + return false, fmt.Errorf("failed to describe Public IPv4 Pool %q: %w", *publicIpv4Pool, err) + } + if len(pools.PublicIpv4Pools) != 1 { + return false, fmt.Errorf("unexpected number of configured Public IPv4 Pools. want 1, got %d", len(pools.PublicIpv4Pools)) + } + + freeIPs := aws.Int64Value(pools.PublicIpv4Pools[0].TotalAvailableAddressCount) + hasFreeIPs := freeIPs >= want + + // force to fallback to Amazon pool when the custom pool is full. + fallbackToAmazonPool := pool.PublicIpv4PoolFallBackOrder != nil && pool.PublicIpv4PoolFallBackOrder.Equal(infrav1.PublicIpv4PoolFallbackOrderAmazonPool) + if !hasFreeIPs && fallbackToAmazonPool { + s.scope.Debug(fmt.Sprintf("public IPv4 pool %q has reached the limit with %d IPs available, using user-defined fallback config %q", *publicIpv4Pool, freeIPs, pool.PublicIpv4PoolFallBackOrder.String()), "eip") + return false, nil + } + if !hasFreeIPs { + return false, fmt.Errorf("public IPv4 pool %q does not have enough free IP addresses: want %d, got %d", *publicIpv4Pool, want, freeIPs) + } + s.scope.Debug(fmt.Sprintf("public IPv4 Pool %q has %d IPs available", *publicIpv4Pool, freeIPs), "eip") + return true, nil +} diff --git a/pkg/cloud/services/network/natgateways.go b/pkg/cloud/services/network/natgateways.go index 4c549a39e5..656bdca208 100644 --- a/pkg/cloud/services/network/natgateways.go +++ b/pkg/cloud/services/network/natgateways.go @@ -222,7 +222,7 @@ func (s *Service) getNatGatewayTagParams(id string) infrav1.BuildParams { } func (s *Service) createNatGateways(subnetIDs []string) (natgateways []*ec2.NatGateway, err error) { - eips, err := s.getOrAllocateAddresses(len(subnetIDs), infrav1.APIServerRoleTagValue) + eips, err := s.getOrAllocateAddresses(len(subnetIDs), infrav1.CommonRoleTagValue, s.scope.VPC().GetElasticIPPool()) if err != nil { return nil, errors.Wrapf(err, "failed to create one or more IP addresses for NAT gateways") } diff --git a/pkg/cloud/services/network/natgateways_test.go b/pkg/cloud/services/network/natgateways_test.go index 29dc45ec13..a77686d61e 100644 --- a/pkg/cloud/services/network/natgateways_test.go +++ b/pkg/cloud/services/network/natgateways_test.go @@ -122,7 +122,7 @@ func TestReconcileNatGateways(t *testing.T) { Tags: []*ec2.Tag{ { Key: aws.String("Name"), - Value: aws.String("test-cluster-eip-apiserver"), + Value: aws.String("test-cluster-eip-common"), }, { Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), @@ -130,7 +130,7 @@ func TestReconcileNatGateways(t *testing.T) { }, { Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), - Value: aws.String("apiserver"), + Value: aws.String("common"), }, }, }, @@ -229,7 +229,7 @@ func TestReconcileNatGateways(t *testing.T) { Tags: []*ec2.Tag{ { Key: aws.String("Name"), - Value: aws.String("test-cluster-eip-apiserver"), + Value: aws.String("test-cluster-eip-common"), }, { Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), @@ -237,7 +237,7 @@ func TestReconcileNatGateways(t *testing.T) { }, { Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), - Value: aws.String("apiserver"), + Value: aws.String("common"), }, }, },