Enabled the property-based scheduling experience

.github/workflows/ci.yml

@@ -116,3 +116,36 @@ jobs:
         env:
           KUBECONFIG: '/home/runner/.kube/config'
           HUB_SERVER_URL: 'https://172.19.0.2:6443'
+
+  e2e-tests-with-aks-property-provider:
+    runs-on: ubuntu-latest
+    needs: [
+      detect-noop,
+    ]
+    if: needs.detect-noop.outputs.noop != 'true'
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Install Ginkgo CLI
+        run: |
+          go install github.com/onsi/ginkgo/v2/[email protected]
+      
+      - name: Install Kind
+        # Before updating the kind version to use, verify that the current kind image
+        # is still supported by the version.
+        run: |
+          go install sigs.k8s.io/[email protected]
+
+      - name: Run e2e tests
+        run: |
+          make e2e-tests
+        env:
+          KUBECONFIG: '/home/runner/.kube/config'
+          HUB_SERVER_URL: 'https://172.19.0.2:6443'
+          PROPERTY_PROVIDER: 'aks'

charts/member-agent/README.md

@@ -40,5 +40,7 @@ helm upgrade member-agent member-agent/ --namespace fleet-system
 | resources                | The resource request/limits for the container image   | limits: "2" CPU, 4Gi, requests: 100m CPU, 128Mi |
 | namespace                | Namespace that this Helm chart is installed on.       | `fleet-system`                                  |
 | logVerbosity             | Log level. Uses V logs (klog)                         | `3`                                             |
+| propertyProvider         | The property provider to use with the member agent    | ``                                              |
+| region                   | The region where the member cluster resides           | ``                                              |
 
 ## Contributing Changes

charts/member-agent/templates/deployment.yaml

@@ -34,6 +34,12 @@ spec:
             - -add_dir_header
             - --enable-v1alpha1-apis={{ .Values.enableV1Alpha1APIs }}
             - --enable-v1beta1-apis={{ .Values.enableV1Beta1APIs }}
+            {{- if .Values.propertyProvider }}
+            - --property-provider={{ .Values.propertyProvider }}
+            {{- end }}
+            {{- if .Values.region }}
+            - --region={{ .Values.region }}
+            {{- end }}
           env:
           - name: HUB_SERVER_URL
             value: "{{ .Values.config.hubURL }}"

cmd/memberagent/main.go

@@ -45,6 +45,8 @@ import (
 	"go.goms.io/fleet/pkg/controllers/work"
 	workv1alpha1controller "go.goms.io/fleet/pkg/controllers/workv1alpha1"
 	fleetmetrics "go.goms.io/fleet/pkg/metrics"
+	"go.goms.io/fleet/pkg/propertyprovider"
+	"go.goms.io/fleet/pkg/propertyprovider/aks"
 	"go.goms.io/fleet/pkg/utils"
 	"go.goms.io/fleet/pkg/utils/httpclient"
 	//+kubebuilder:scaffold:imports
@@ -63,6 +65,8 @@ var (
 	leaderElectionNamespace = flag.String("leader-election-namespace", "kube-system", "The namespace in which the leader election resource will be created.")
 	enableV1Alpha1APIs      = flag.Bool("enable-v1alpha1-apis", true, "If set, the agents will watch for the v1alpha1 APIs.")
 	enableV1Beta1APIs       = flag.Bool("enable-v1beta1-apis", false, "If set, the agents will watch for the v1beta1 APIs.")
+	propertyProvider        = flag.String("property-provider", "none", "The property provider to use for the agent.")
+	region                  = flag.String("region", "", "The region where the member cluster resides.")
 )
 
 func init() {
@@ -339,14 +343,35 @@ func Start(ctx context.Context, hubCfg, memberConfig *rest.Config, hubOpts, memb
 		}
 
 		klog.Info("Setting up the internalMemberCluster v1beta1 controller")
-		imcReconciler, err := imcv1beta1.NewReconciler(ctx, hubMgr.GetClient(), memberMgr.GetConfig(), memberMgr.GetClient(), workController, nil)
+		// Set up a provider provider (if applicable).
+		var pp propertyprovider.PropertyProvider
+		switch {
+		case propertyProvider != nil && *propertyProvider == "aks":
+			klog.V(2).Info("setting up the AKS property provider")
+			// Note that the property provider, though initialized here, is not started until
+			// the specific instance wins the leader election.
+			pp = aks.New(region)
+		default:
+			// Fall back to not using any property provider if the provided type is none or
+			// not recognizable.
+			klog.V(2).Info("no property provider is specified, or the given type is not recognizable; start with no property provider")
+			pp = nil
+		}
+
+		// Set up the IMC controller.
+		imcReconciler, err := imcv1beta1.NewReconciler(
+			ctx,
+			hubMgr.GetClient(),
+			memberMgr.GetConfig(), memberMgr.GetClient(),
+			workController,
+			pp)
 		if err != nil {
-			klog.ErrorS(err, "Failed to create v1beta1 controller", "controller", "internalMemberCluster")
-			return fmt.Errorf("unable to create internalMemberCluster v1beta1 controller: %w", err)
+			klog.ErrorS(err, "Failed to create internalMemberCluster v1beta1 reconciler")
+			return fmt.Errorf("failed to create internalMemberCluster v1beta1 reconciler: %w", err)
 		}
 		if err := imcReconciler.SetupWithManager(hubMgr); err != nil {
-			klog.ErrorS(err, "Failed to set up v1beta1 controller with controller manager", "controller", "internalMemberCluster")
-			return fmt.Errorf("unable to set up internalMemberCluster v1beta1 controller with controller manager: %w", err)
+			klog.ErrorS(err, "Failed to create v1beta1 controller", "controller", "internalMemberCluster")
+			return fmt.Errorf("failed to create internalMemberCluster v1beta1 controller: %w", err)
 		}
 	}
 

pkg/propertyprovider/aks/provider.go

@@ -119,7 +119,7 @@ func (p *PropertyProvider) Start(ctx context.Context, config *rest.Config) error
 		return err
 	}
 
-	if p.region == nil {
+	if p.region == nil || len(*p.region) == 0 {
 		klog.V(2).Info("Auto-discover region as none has been specified")
 		// Note that an API reader is passed here for the purpose of auto-discovering region
 		// information from AKS nodes; at this time the cache from the controller manager
@@ -134,7 +134,7 @@ func (p *PropertyProvider) Start(ctx context.Context, config *rest.Config) error
 		}
 		p.region = discoveredRegion
 	}
-	klog.V(2).Infof("Starting with the specified region %s", *p.region)
+	klog.V(2).Infof("Starting with the region set to %s", *p.region)
 	pp := trackers.NewAKSKarpenterPricingClient(ctx, *p.region)
 	p.podTracker = trackers.NewPodTracker()
 	p.nodeTracker = trackers.NewNodeTracker(pp)

test/e2e/framework/cluster.go

@@ -17,6 +17,8 @@ import (
 	"k8s.io/client-go/tools/clientcmd/api"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+
+	"go.goms.io/fleet/pkg/propertyprovider/aks/trackers"
 )
 
 var (
@@ -33,13 +35,15 @@ type Cluster struct {
 	PresentingServiceAccountInHubClusterName string
 	HubURL                                   string
 	RestMapper                               meta.RESTMapper
+	PricingProvider                          trackers.PricingProvider
 }
 
-func NewCluster(name, svcAccountName string, scheme *runtime.Scheme) *Cluster {
+func NewCluster(name, svcAccountName string, scheme *runtime.Scheme, pp trackers.PricingProvider) *Cluster {
 	return &Cluster{
 		Scheme:                                   scheme,
 		ClusterName:                              name,
 		PresentingServiceAccountInHubClusterName: svcAccountName,
+		PricingProvider:                          pp,
 	}
 }
 

test/e2e/kindconfigs/cluster-1.yaml

@@ -0,0 +1,9 @@
+# Important: due to kind limitations
+# (kind node always has resource capacity == host resource capacity) and the way our planned test
+# cases (resource and non-resource properties) are designed, modification of this setup might lead
+# to test failures.
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker

test/e2e/kindconfigs/cluster-2.yaml

@@ -0,0 +1,10 @@
+# Important: due to kind limitations
+# (kind node always has resource capacity == host resource capacity) and the way our planned test
+# cases (resource and non-resource properties) are designed, modification of this setup might lead
+# to test failures.
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker

test/e2e/kindconfigs/cluster-3.yaml

@@ -0,0 +1,11 @@
+# Important: due to kind limitations
+# (kind node always has resource capacity == host resource capacity) and the way our planned test
+# cases (resource and non-resource properties) are designed, modification of this setup might lead
+# to test failures.
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
+- role: worker