From 17a0be19b2d5af129d2077fba59fb54f119d4c4e Mon Sep 17 00:00:00 2001
From: Charlie McBride <charlie.mcbride@microsoft.com>
Date: Wed, 2 Oct 2024 10:12:48 -0700
Subject: [PATCH] add CiliumAgentNotReadyTaint to precent overprovisioning

---
 examples/v1beta1/general-purpose.yaml     | 3 +++
 examples/v1beta1/multi-arch.yaml          | 3 +++
 examples/v1beta1/system-surge.yaml        | 3 +++
 test/pkg/environment/azure/environment.go | 9 ++++++++-
 4 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/examples/v1beta1/general-purpose.yaml b/examples/v1beta1/general-purpose.yaml
index 5e11a604e..c11ae0151 100644
--- a/examples/v1beta1/general-purpose.yaml
+++ b/examples/v1beta1/general-purpose.yaml
@@ -11,6 +11,9 @@ spec:
     expireAfter: Never
   template:
     spec:
+      startupTaints:
+        - key: node.kubernetes.io/network-unavailable
+          effect: NoSchedule
       requirements:
         - key: kubernetes.io/arch
           operator: In
diff --git a/examples/v1beta1/multi-arch.yaml b/examples/v1beta1/multi-arch.yaml
index d5bf1438c..cde0f9213 100644
--- a/examples/v1beta1/multi-arch.yaml
+++ b/examples/v1beta1/multi-arch.yaml
@@ -11,6 +11,9 @@ spec:
     expireAfter: Never
   template:
     spec:
+      startupTaints:
+        - key: node.kubernetes.io/network-unavailable
+          effect: NoSchedule
       requirements:
         - key: kubernetes.io/arch
           operator: In
diff --git a/examples/v1beta1/system-surge.yaml b/examples/v1beta1/system-surge.yaml
index 267008b3c..0ee0f8ba0 100644
--- a/examples/v1beta1/system-surge.yaml
+++ b/examples/v1beta1/system-surge.yaml
@@ -14,6 +14,9 @@ spec:
       labels: 
         kubernetes.azure.com/mode: "system"
     spec:
+      startupTaints:
+        - key: node.kubernetes.io/network-unavailable
+          effect: NoSchedule
       taints: 
         - key: "CriticalAddonsOnly"
           value: "true"
diff --git a/test/pkg/environment/azure/environment.go b/test/pkg/environment/azure/environment.go
index f5797b2f6..3676bf922 100644
--- a/test/pkg/environment/azure/environment.go
+++ b/test/pkg/environment/azure/environment.go
@@ -37,7 +37,10 @@ func init() {
 	corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": v1.LabelTopologyZone})
 }
 
-const WindowsDefaultImage = "mcr.microsoft.com/oss/kubernetes/pause:3.9"
+const (
+	WindowsDefaultImage      = "mcr.microsoft.com/oss/kubernetes/pause:3.9"
+	CiliumAgentNotReadyTaint = "node.cilium.io/agent-not-ready"
+)
 
 type Environment struct {
 	*common.Environment
@@ -89,6 +92,10 @@ func (env *Environment) DefaultNodePool(nodeClass *v1alpha2.AKSNodeClass) *corev
 		v1.ResourceCPU:    resource.MustParse("100"),
 		v1.ResourceMemory: resource.MustParse("1000Gi"),
 	})
+	nodePool.Spec.Template.Spec.StartupTaints = append(nodePool.Spec.Template.Spec.StartupTaints, v1.Taint{
+		Key:    CiliumAgentNotReadyTaint,
+		Effect: v1.TaintEffectNoSchedule,
+	})
 	return nodePool
 }