Add support for joining cluster to AKS Fleet

Makefile

@@ -158,7 +158,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook
 RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
 ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml
 ASO_VERSION := v2.4.0
-ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com
+ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com fleets.containerservice.azure.com fleetsmembers.containerservice.azure.com
 
 # Allow overriding the imagePullPolicy
 PULL_POLICY ?= Always

Tiltfile

@@ -22,7 +22,7 @@ settings = {
     "capi_version": "v1.5.3",
     "cert_manager_version": "v1.13.1",
     "kubernetes_version": "v1.28.0",
-    "aks_kubernetes_version": "v1.26.3",
+    "aks_kubernetes_version": "v1.27.3",
     "flatcar_version": "3374.2.1",
 }
 

api/v1beta1/azuremanagedcontrolplane_default.go

@@ -98,6 +98,13 @@ func (m *AzureManagedControlPlane) setDefaultSubnet() {
 	}
 }
 
+// setDefaultFleetsMenber sets the default FleetsMember for an AzureManagedControlPlane.
+func (m *AzureManagedControlPlane) setDefaultFleetsMember() {
+	if clusterName, ok := m.Labels[clusterv1.ClusterNameLabel]; ok && m.Spec.FleetsMember != nil && m.Spec.FleetsMember.Name == "" {
+		m.Spec.FleetsMember.Name = clusterName
+	}
+}
+
 func setDefaultSku(sku *AKSSku) *AKSSku {
 	result := sku.DeepCopy()
 	if sku == nil {

api/v1beta1/azuremanagedcontrolplane_types.go

@@ -110,6 +110,10 @@ type AzureManagedControlPlaneSpec struct {
 	// Immutable.
 	// +optional
 	DNSPrefix *string `json:"dnsPrefix,omitempty"`
+
+	// FleetsMember is the spec for the fleet this cluster is a member of.
+	// +optional
+	FleetsMember *FleetsMember `json:"fleetManager,omitempty"`
 }
 
 // HTTPProxyConfig is the HTTP proxy configuration for the cluster.

api/v1beta1/azuremanagedcontrolplane_webhook.go

@@ -94,6 +94,7 @@ func (mw *azureManagedControlPlaneWebhook) Default(ctx context.Context, obj runt
 	m.setDefaultSubnet()
 	m.setDefaultOIDCIssuerProfile()
 	m.setDefaultDNSPrefix()
+	m.setDefaultFleetsMember()
 
 	return nil
 }

api/v1beta1/azuremanagedcontrolplane_webhook_test.go

@@ -37,6 +37,9 @@ func TestDefaultingWebhook(t *testing.T) {
 	amcp := &AzureManagedControlPlane{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "fooName",
+			Labels: map[string]string{
+				clusterv1.ClusterNameLabel: "fooCluster",
+			},
 		},
 		Spec: AzureManagedControlPlaneSpec{
 			AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{
@@ -80,6 +83,7 @@ func TestDefaultingWebhook(t *testing.T) {
 		Enabled: ptr.To(true),
 	}
 	amcp.Spec.DNSPrefix = ptr.To("test-prefix")
+	amcp.Spec.FleetsMember = &FleetsMember{}
 
 	err = mcpw.Default(context.Background(), amcp)
 	g.Expect(err).NotTo(HaveOccurred())
@@ -94,6 +98,8 @@ func TestDefaultingWebhook(t *testing.T) {
 	g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeTrue())
 	g.Expect(amcp.Spec.DNSPrefix).ToNot(BeNil())
 	g.Expect(*amcp.Spec.DNSPrefix).To(Equal("test-prefix"))
+	g.Expect(amcp.Spec.FleetsMember.Name).To(Equal("fooCluster"))
+
 	t.Logf("Testing amcp defaulting webhook with overlay")
 	amcp = &AzureManagedControlPlane{
 		ObjectMeta: metav1.ObjectMeta{

api/v1beta1/consts.go

@@ -130,6 +130,8 @@ const (
 	NetworkInterfaceReadyCondition clusterv1.ConditionType = "NetworkInterfacesReady"
 	// PrivateEndpointsReadyCondition means the private endpoints exist and are ready to be used.
 	PrivateEndpointsReadyCondition clusterv1.ConditionType = "PrivateEndpointsReady"
+	// FleetReadyCondition means the Fleet exists and is ready to be used.
+	FleetReadyCondition clusterv1.ConditionType = "FleetReady"
 
 	// CreatingReason means the resource is being created.
 	CreatingReason = "Creating"

api/v1beta1/types.go

@@ -1005,6 +1005,25 @@ type AzureBastion struct {
 	EnableTunneling bool `json:"enableTunneling,omitempty"`
 }
 
+// FleetsMember defines the fleets member configuration.
+type FleetsMember struct {
+	// Name is the name of the member.
+	// +optional
+	Name string `json:"name,omitempty"`
+
+	// Group is the group this member belongs to for multi-cluster update management.
+	// +optional
+	Group string `json:"group,omitempty"`
+
+	// ManagerName is the name of the fleet manager.
+	// +kubebuilder:validation:Required
+	ManagerName string `json:"managerName,omitempty"`
+
+	// ManagerResourceGroup is the resource group of the fleet manager.
+	// +kubebuilder:validation:Required
+	ManagerResourceGroup string `json:"managerResourceGroup,omitempty"`
+}
+
 // BackendPool describes the backend pool of the load balancer.
 type BackendPool struct {
 	// Name specifies the name of backend pool for the load balancer. If not specified, the default name will

azure/defaults.go

@@ -295,6 +295,11 @@ func ManagedClusterID(subscriptionID, resourceGroup, managedClusterName string)
 	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerService/managedClusters/%s", subscriptionID, resourceGroup, managedClusterName)
 }
 
+// FleetID returns the azure resource ID for a given fleet manager.
+func FleetID(subscriptionID, resourceGroup, fleetName string) string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerService/fleets/%s", subscriptionID, resourceGroup, fleetName)
+}
+
 // GetBootstrappingVMExtension returns the CAPZ Bootstrapping VM extension.
 // The CAPZ Bootstrapping extension is a simple clone of https://github.com/Azure/custom-script-extension-linux for Linux or
 // https://learn.microsoft.com/azure/virtual-machines/extensions/custom-script-windows for Windows.

azure/scope/managedcontrolplane.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230201"
+	asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview"
 	asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601"
 	"github.com/pkg/errors"
 	"golang.org/x/mod/semver"
@@ -36,6 +37,7 @@ import (
 	"k8s.io/utils/ptr"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/azure"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/services/fleetsmember"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/groups"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints"
@@ -213,6 +215,11 @@ func (s *ManagedControlPlaneScope) AdditionalTags() infrav1.Tags {
 	return tags
 }
 
+// AzureFleetMembership returns the cluster AzureFleetMembership.
+func (s *ManagedControlPlaneScope) AzureFleetMembership() *infrav1.FleetsMember {
+	return s.ControlPlane.Spec.FleetsMember
+}
+
 // SubscriptionID returns the Azure client Subscription ID.
 func (s *ManagedControlPlaneScope) SubscriptionID() string {
 	return s.AzureClients.SubscriptionID()
@@ -289,6 +296,23 @@ func (s *ManagedControlPlaneScope) VNetSpec() azure.ResourceSpecGetter {
 	}
 }
 
+// AzureFleetsMemberSpec returns the fleet spec.
+func (s *ManagedControlPlaneScope) AzureFleetsMemberSpec() azure.ASOResourceSpecGetter[*asocontainerservicev1preview.FleetsMember] {
+	if s.AzureFleetMembership() == nil {
+		return nil
+	}
+	return &fleetsmember.AzureFleetsMemberSpec{
+		Name:                 s.AzureFleetMembership().Name,
+		Namespace:            s.Cluster.Namespace,
+		ClusterName:          s.ClusterName(),
+		ClusterResourceGroup: s.ResourceGroup(),
+		Group:                s.AzureFleetMembership().Group,
+		SubscriptionID:       s.SubscriptionID(),
+		ManagerName:          s.AzureFleetMembership().ManagerName,
+		ManagerResourceGroup: s.AzureFleetMembership().ManagerResourceGroup,
+	}
+}
+
 // ControlPlaneRouteTable returns the cluster controlplane routetable.
 func (s *ManagedControlPlaneScope) ControlPlaneRouteTable() infrav1.RouteTable {
 	return infrav1.RouteTable{}

azure/services/fleetsmember/fleetsmember.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fleetsmember
+
+import (
+	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview"
+	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-azure/azure"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/services/aso"
+)
+
+const serviceName = "fleetsmember"
+
+// FleetsMemberScope defines the scope interface for a Fleet host service.
+type FleetsMemberScope interface {
+	azure.ClusterScoper
+	aso.Scope
+	AzureFleetsMemberSpec() azure.ASOResourceSpecGetter[*asocontainerservicev1.FleetsMember]
+}
+
+// Service provides operations on Azure resources.
+type Service struct {
+	Scope FleetsMemberScope
+	*aso.Service[*asocontainerservicev1.FleetsMember, FleetsMemberScope]
+}
+
+// New creates a new service.
+func New(scope FleetsMemberScope) *Service {
+	svc := aso.NewService[*asocontainerservicev1.FleetsMember, FleetsMemberScope](serviceName, scope)
+	spec := scope.AzureFleetsMemberSpec()
+	if spec != nil {
+		svc.Specs = []azure.ASOResourceSpecGetter[*asocontainerservicev1.FleetsMember]{spec}
+	}
+	svc.ConditionType = infrav1.FleetReadyCondition
+	return &Service{
+		Scope:   scope,
+		Service: svc,
+	}
+}

azure/services/fleetsmember/spec.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fleetsmember
+
+import (
+	"context"
+
+	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview"
+	"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/cluster-api-provider-azure/azure"
+)
+
+// AzureFleetsMemberSpec defines the specification for an Azure Fleets Member.
+type AzureFleetsMemberSpec struct {
+	Name                 string
+	Namespace            string
+	ClusterName          string
+	ClusterResourceGroup string
+	Group                string
+	SubscriptionID       string
+	ManagerName          string
+	ManagerResourceGroup string
+}
+
+// ResourceRef implements azure.ASOResourceSpecGetter.
+func (s *AzureFleetsMemberSpec) ResourceRef() *asocontainerservicev1.FleetsMember {
+	return &asocontainerservicev1.FleetsMember{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      s.Name,
+			Namespace: s.Namespace,
+		},
+	}
+}
+
+// Parameters implements azure.ASOResourceSpecGetter.
+func (s *AzureFleetsMemberSpec) Parameters(ctx context.Context, existingFleetsMember *asocontainerservicev1.FleetsMember) (parameters *asocontainerservicev1.FleetsMember, err error) {
+	if existingFleetsMember != nil {
+		return existingFleetsMember, nil
+	}
+
+	fleetsMember := &asocontainerservicev1.FleetsMember{}
+	fleetsMember.Spec = asocontainerservicev1.Fleets_Member_Spec{}
+	fleetsMember.Spec.AzureName = s.Name
+	fleetsMember.Spec.Owner = &genruntime.KnownResourceReference{
+		ARMID: azure.FleetID(s.SubscriptionID, s.ManagerResourceGroup, s.ManagerName),
+	}
+	fleetsMember.Spec.Group = ptr.To(s.Group)
+	fleetsMember.Spec.ClusterResourceReference = &genruntime.ResourceReference{
+		ARMID: azure.ManagedClusterID(s.SubscriptionID, s.ClusterResourceGroup, s.ClusterName),
+	}
+
+	return fleetsMember, nil
+}
+
+// WasManaged implements azure.ASOResourceSpecGetter.
+func (s *AzureFleetsMemberSpec) WasManaged(resource *asocontainerservicev1.FleetsMember) bool {
+	// returns always returns true as CAPZ does not support BYO fleet.
+	return true
+}