Merge pull request #4882 from mtulio/CORS-2899-edge-zones

✨  Introduce edge subnets to support AWS Local Zones

api/v1beta1/awscluster_conversion.go

@@ -104,14 +104,19 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.NetworkSpec.VPC.EmptyRoutesDefaultVPCSecurityGroup = restored.Spec.NetworkSpec.VPC.EmptyRoutesDefaultVPCSecurityGroup
 	dst.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch = restored.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch
 
-	// Restore SubnetSpec.ResourceID field, if any.
+	// Restore SubnetSpec.ResourceID, SubnetSpec.ParentZoneName, and SubnetSpec.ZoneType fields, if any.
 	for _, subnet := range restored.Spec.NetworkSpec.Subnets {
-		if len(subnet.ResourceID) == 0 {
-			continue
-		}
 		for i, dstSubnet := range dst.Spec.NetworkSpec.Subnets {
 			if dstSubnet.ID == subnet.ID {
-				dstSubnet.ResourceID = subnet.ResourceID
+				if len(subnet.ResourceID) > 0 {
+					dstSubnet.ResourceID = subnet.ResourceID
+				}
+				if subnet.ParentZoneName != nil {
+					dstSubnet.ParentZoneName = subnet.ParentZoneName
+				}
+				if subnet.ZoneType != nil {
+					dstSubnet.ZoneType = subnet.ZoneType
+				}
 				dstSubnet.DeepCopyInto(&dst.Spec.NetworkSpec.Subnets[i])
 			}
 		}

api/v1beta2/awscluster_webhook.go

@@ -248,6 +248,11 @@ func (r *AWSCluster) validateNetwork() field.ErrorList {
 		if subnet.IsIPv6 || subnet.IPv6CidrBlock != "" {
 			allErrs = append(allErrs, field.Invalid(field.NewPath("subnets"), r.Spec.NetworkSpec.Subnets, "IPv6 cannot be used with unmanaged clusters at this time."))
 		}
+		if subnet.ZoneType != nil && subnet.IsEdge() {
+			if subnet.ParentZoneName == nil {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("subnets"), r.Spec.NetworkSpec.Subnets, "ParentZoneName must be set when ZoneType is 'local-zone'."))
+			}
+		}
 	}
 
 	if r.Spec.NetworkSpec.VPC.CidrBlock != "" && r.Spec.NetworkSpec.VPC.IPAMPool != nil {

api/v1beta2/network_types.go

@@ -20,6 +20,10 @@ import (
 	"fmt"
 	"sort"
 	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"k8s.io/utils/ptr"
 )
 
 const (
@@ -37,6 +41,11 @@ const (
 	DefaultAPIServerHealthThresholdCount = 5
 	// DefaultAPIServerUnhealthThresholdCount the API server unhealthy check threshold count.
 	DefaultAPIServerUnhealthThresholdCount = 3
+
+	// ZoneTypeAvailabilityZone defines the regular AWS zones in the Region.
+	ZoneTypeAvailabilityZone ZoneType = "availability-zone"
+	// ZoneTypeLocalZone defines the AWS zone type in Local Zone infrastructure.
+	ZoneTypeLocalZone ZoneType = "local-zone"
 )
 
 // NetworkStatus encapsulates AWS networking resources.
@@ -508,6 +517,39 @@ type SubnetSpec struct {
 
 	// Tags is a collection of tags describing the resource.
 	Tags Tags `json:"tags,omitempty"`
+
+	// ZoneType defines the type of the zone where the subnet is created.
+	//
+	// The valid values are availability-zone, and local-zone.
+	//
+	// Subnet with zone type availability-zone (regular) is always selected to create cluster
+	// resources, like Load Balancers, NAT Gateways, Contol Plane nodes, etc.
+	//
+	// Subnet with zone type local-zone is not eligible to automatically create
+	// regular cluster resources.
+	//
+	// The public subnet in availability-zone or local-zone is associated with regular public
+	// route table with default route entry to a Internet Gateway.
+	//
+	// The private subnet in the availability-zone is associated with a private route table with
+	// the default route entry to a NAT Gateway created in that zone.
+	//
+	// The private subnet in the local-zone is associated with a private route table with
+	// the default route entry re-using the NAT Gateway in the Region (preferred from the
+	// parent zone, the zone type availability-zone in the region, or first table available).
+	//
+	// +kubebuilder:validation:Enum=availability-zone;local-zone
+	// +optional
+	ZoneType *ZoneType `json:"zoneType,omitempty"`
+
+	// ParentZoneName is the zone name where the current subnet's zone is tied when
+	// the zone is a Local Zone.
+	//
+	// The subnets in Local Zone locations consume the ParentZoneName to determine the correct
+	// private route table to egress traffic to the internet.
+	//
+	// +optional
+	ParentZoneName *string `json:"parentZoneName,omitempty"`
 }
 
 // GetResourceID returns the identifier for this subnet,
@@ -524,6 +566,39 @@ func (s *SubnetSpec) String() string {
 	return fmt.Sprintf("id=%s/az=%s/public=%v", s.GetResourceID(), s.AvailabilityZone, s.IsPublic)
 }
 
+// IsEdge returns the true when the subnet is created in the edge zone,
+// Local Zones.
+func (s *SubnetSpec) IsEdge() bool {
+	return s.ZoneType != nil && *s.ZoneType == ZoneTypeLocalZone
+}
+
+// SetZoneInfo updates the subnets with zone information.
+func (s *SubnetSpec) SetZoneInfo(zones []*ec2.AvailabilityZone) error {
+	zoneInfo := func(zoneName string) *ec2.AvailabilityZone {
+		for _, zone := range zones {
+			if aws.StringValue(zone.ZoneName) == zoneName {
+				return zone
+			}
+		}
+		return nil
+	}
+
+	zone := zoneInfo(s.AvailabilityZone)
+	if zone == nil {
+		if len(s.AvailabilityZone) > 0 {
+			return fmt.Errorf("unable to update zone information for subnet '%v' and zone '%v'", s.ID, s.AvailabilityZone)
+		}
+		return fmt.Errorf("unable to update zone information for subnet '%v'", s.ID)
+	}
+	if zone.ZoneType != nil {
+		s.ZoneType = ptr.To(ZoneType(*zone.ZoneType))
+	}
+	if zone.ParentZoneName != nil {
+		s.ParentZoneName = zone.ParentZoneName
+	}
+	return nil
+}
+
 // Subnets is a slice of Subnet.
 // +listType=map
 // +listMapKey=id
@@ -541,6 +616,22 @@ func (s Subnets) ToMap() map[string]*SubnetSpec {
 
 // IDs returns a slice of the subnet ids.
 func (s Subnets) IDs() []string {
+	res := []string{}
+	for _, subnet := range s {
+		// Prevent returning edge zones (Local Zone) to regular Subnet IDs.
+		// Edge zones should not deploy control plane nodes, and does not support Nat Gateway and
+		// Network Load Balancers. Any resource for the core infrastructure should not consume edge
+		// zones.
+		if subnet.IsEdge() {
+			continue
+		}
+		res = append(res, subnet.GetResourceID())
+	}
+	return res
+}
+
+// IDsWithEdge returns a slice of the subnet ids.
+func (s Subnets) IDsWithEdge() []string {
 	res := []string{}
 	for _, subnet := range s {
 		res = append(res, subnet.GetResourceID())
@@ -581,21 +672,29 @@ func (s Subnets) FindEqual(spec *SubnetSpec) *SubnetSpec {
 // FilterPrivate returns a slice containing all subnets marked as private.
 func (s Subnets) FilterPrivate() (res Subnets) {
 	for _, x := range s {
+		// Subnets in AWS Local Zones or Wavelength should not be used by core infrastructure.
+		if x.IsEdge() {
+			continue
+		}
 		if !x.IsPublic {
 			res = append(res, x)
 		}
 	}
-	return
+	return res
 }
 
 // FilterPublic returns a slice containing all subnets marked as public.
 func (s Subnets) FilterPublic() (res Subnets) {
 	for _, x := range s {
+		// Subnets in AWS Local Zones or Wavelength should not be used by core infrastructure.
+		if x.IsEdge() {
+			continue
+		}
 		if x.IsPublic {
 			res = append(res, x)
 		}
 	}
-	return
+	return res
 }
 
 // FilterByZone returns a slice containing all subnets that live in the availability zone specified.
@@ -605,22 +704,32 @@ func (s Subnets) FilterByZone(zone string) (res Subnets) {
 			res = append(res, x)
 		}
 	}
-	return
+	return res
 }
 
 // GetUniqueZones returns a slice containing the unique zones of the subnets.
 func (s Subnets) GetUniqueZones() []string {
 	keys := make(map[string]bool)
 	zones := []string{}
 	for _, x := range s {
-		if _, value := keys[x.AvailabilityZone]; !value {
+		if _, value := keys[x.AvailabilityZone]; len(x.AvailabilityZone) > 0 && !value {
 			keys[x.AvailabilityZone] = true
 			zones = append(zones, x.AvailabilityZone)
 		}
 	}
 	return zones
 }
 
+// SetZoneInfo updates the subnets with zone information.
+func (s Subnets) SetZoneInfo(zones []*ec2.AvailabilityZone) error {
+	for i := range s {
+		if err := s[i].SetZoneInfo(zones); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // CNISpec defines configuration for CNI.
 type CNISpec struct {
 	// CNIIngressRules specify rules to apply to control plane and worker node security groups.
@@ -835,3 +944,11 @@ func (i *IngressRule) Equals(o *IngressRule) bool {
 
 	return true
 }
+
+// ZoneType defines listener AWS Availability Zone type.
+type ZoneType string
+
+// String returns the string representation for the zone type.
+func (z ZoneType) String() string {
+	return string(z)
+}