Use separate default feature gate

apis/apps/v1/petset_webhook.go

@@ -22,7 +22,6 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -67,14 +66,14 @@ func (obj *PetSet) Default() {
 		if obj.Spec.UpdateStrategy.RollingUpdate.Partition == nil {
 			obj.Spec.UpdateStrategy.RollingUpdate.Partition = ptr.To[int32](0)
 		}
-		if utilfeature.DefaultFeatureGate.Enabled(features.MaxUnavailablePetSet) {
+		if features.DefaultFeatureGate.Enabled(features.MaxUnavailablePetSet) {
 			if obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable == nil {
 				obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable = ptr.To(intstr.FromInt32(1))
 			}
 		}
 	}
 
-	if utilfeature.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
+	if features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
 		if obj.Spec.PersistentVolumeClaimRetentionPolicy == nil {
 			obj.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{}
 		}

go.mod

@@ -15,7 +15,6 @@ require (
 	gomodules.xyz/pointer v0.1.0
 	gomodules.xyz/x v0.0.17
 	k8s.io/api v0.30.1
-	k8s.io/apiextensions-apiserver v0.30.1
 	k8s.io/apimachinery v0.30.1
 	k8s.io/apiserver v0.30.1
 	k8s.io/client-go v0.30.1
@@ -176,6 +175,7 @@ require (
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.30.1 // indirect
 	k8s.io/cli-runtime v0.30.1 // indirect
 	k8s.io/cloud-provider v0.30.1 // indirect
 	k8s.io/component-helpers v0.30.1 // indirect

pkg/cmds/server/operator.go

@@ -25,6 +25,7 @@ import (
 	apiinformers "kubeops.dev/petset/client/informers/externalversions"
 	"kubeops.dev/petset/pkg/controller"
 	"kubeops.dev/petset/pkg/controller/petset"
+	"kubeops.dev/petset/pkg/features"
 
 	"github.com/spf13/pflag"
 	"k8s.io/client-go/informers"
@@ -64,6 +65,7 @@ func (s *OperatorOptions) AddFlags(fs *pflag.FlagSet) {
 	pfs := flag.NewFlagSet("extra-flags", flag.ExitOnError)
 	s.AddGoFlags(pfs)
 	fs.AddGoFlagSet(pfs)
+	features.DefaultMutableFeatureGate.AddFlag(fs)
 }
 
 func (s *OperatorOptions) ApplyTo(cfg *controller.OperatorConfig) error {

pkg/controller/controller_utils.go

@@ -50,7 +50,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/apimachinery/pkg/util/wait"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
@@ -850,7 +849,7 @@ func (s ActivePodsWithRanks) Less(i, j int) bool {
 	}
 
 	// 4. lower pod-deletion-cost < higher pod-deletion cost
-	if utilfeature.DefaultFeatureGate.Enabled(features.PodDeletionCost) {
+	if features.DefaultFeatureGate.Enabled(features.PodDeletionCost) {
 		pi, _ := helper.GetDeletionCostFromPodAnnotations(s.Pods[i].Annotations)
 		pj, _ := helper.GetDeletionCostFromPodAnnotations(s.Pods[j].Annotations)
 		if pi != pj {
@@ -873,7 +872,7 @@ func (s ActivePodsWithRanks) Less(i, j int) bool {
 		readyTime1 := podReadyTime(s.Pods[i])
 		readyTime2 := podReadyTime(s.Pods[j])
 		if !readyTime1.Equal(readyTime2) {
-			if !utilfeature.DefaultFeatureGate.Enabled(features.LogarithmicScaleDown) {
+			if !features.DefaultFeatureGate.Enabled(features.LogarithmicScaleDown) {
 				return afterOrZero(readyTime1, readyTime2)
 			} else {
 				if s.Now.IsZero() || readyTime1.IsZero() || readyTime2.IsZero() {
@@ -893,7 +892,7 @@ func (s ActivePodsWithRanks) Less(i, j int) bool {
 	}
 	// 8. Empty creation time pods < newer pods < older pods
 	if !s.Pods[i].CreationTimestamp.Equal(&s.Pods[j].CreationTimestamp) {
-		if !utilfeature.DefaultFeatureGate.Enabled(features.LogarithmicScaleDown) {
+		if !features.DefaultFeatureGate.Enabled(features.LogarithmicScaleDown) {
 			return afterOrZero(&s.Pods[i].CreationTimestamp, &s.Pods[j].CreationTimestamp)
 		} else {
 			if s.Now.IsZero() || s.Pods[i].CreationTimestamp.IsZero() || s.Pods[j].CreationTimestamp.IsZero() {

pkg/controller/petset/pet_pod_control.go

@@ -1,4 +1,4 @@
 /*
 Copyright 2016 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -31,7 +31,6 @@
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	errorutils "k8s.io/apimachinery/pkg/util/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientset "k8s.io/client-go/kubernetes"
 	corelisters "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/record"
@@ -141,7 +140,7 @@
 	if apierrors.IsAlreadyExists(err) {
 		return err
 	}
-	if utilfeature.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
+	if features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
 		// Set PVC policy as much as is possible at this point.
 		if err := spc.UpdatePodClaimForRetentionPolicy(ctx, set, pod); err != nil {
 			spc.recordPodEvent("update", set, pod, err)
@@ -172,7 +171,7 @@
 				return err
 			}
 		}
-		if utilfeature.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
+		if features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
 			// if the Pod's PVCs are not consistent with the PetSet's PVC deletion policy, update the PVC
 			// and dirty the pod.
 			if match, err := spc.ClaimsMatchRetentionPolicy(ctx, set, pod); err != nil {
@@ -340,7 +339,7 @@
 		return err
 	}
 
-	if utilfeature.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
+	if features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
 		// Set PVC policy as much as is possible at this point.
 		if err := spc.UpdatePodClaimForRetentionPolicy(ctx, set, pod); err != nil {
 			spc.recordPodEvent("update", set, pod, err)

pkg/controller/petset/pet_pod_control_test.go

@@ -48,11 +48,11 @@
 func TestStatefulPodControlCreatesPods(t *testing.T) {
 	recorder := record.NewFakeRecorder(10)
 	set := newPetSet(3)
 	pod := newPetSetPod(set, 0)
 	fakeClient := &fake.Clientset{}
 	claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
 	claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer)
 	control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder)
 	fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
 		return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), action.GetResource().Resource)
 	})
@@ -82,7 +82,7 @@
 func TestStatefulPodControlCreatePodExists(t *testing.T) {
 	recorder := record.NewFakeRecorder(10)
 	set := newPetSet(3)
 	pod := newPetSetPod(set, 0)
 	fakeClient := &fake.Clientset{}
 	pvcs := getPersistentVolumeClaims(set, pod)
 	pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
@@ -567,7 +567,7 @@
 		if err := control.UpdatePodClaimForRetentionPolicy(ctx, set, pod); err != nil {
 			t.Errorf("Unexpected error for UpdatePodClaimForRetentionPolicy (retain): %v", err)
 		}
-		expectRef := utilfeature.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC)
+		expectRef := features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC)
 		for k := range claims {
 			claim, err := claimLister.PersistentVolumeClaims(claims[k].Namespace).Get(claims[k].Name)
 			if err != nil {
@@ -725,7 +725,7 @@
 			}
 		}
 		events := collectEvents(recorder.Events)
-		if utilfeature.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
+		if features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
 			if eventCount := len(events); eventCount != 1 {
 				t.Errorf("delete failed: got %d events, but want 1", eventCount)
 			}

pkg/controller/petset/pet_set_control.go

@@ -31,7 +31,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/integer"
@@ -424,7 +423,7 @@ func (ssc *defaultPetSetControl) processReplica(
 	}
 	// If we find a Pod that has not been created we create the Pod
 	if !isCreated(replicas[i]) {
-		if utilfeature.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
+		if features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
 			if isStale, err := ssc.podControl.PodClaimIsStale(set, replicas[i]); err != nil {
 				return true, err
 			} else if isStale {
@@ -479,7 +478,7 @@ func (ssc *defaultPetSetControl) processReplica(
 
 	// Enforce the PetSet invariants
 	retentionMatch := true
-	if utilfeature.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
+	if features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
 		var err error
 		retentionMatch, err = ssc.podControl.ClaimsMatchRetentionPolicy(ctx, updateSet, replicas[i])
 		// An error is expected if the pod is not yet fully updated, and so return is treated as matching.
@@ -665,7 +664,7 @@ func (ssc *defaultPetSetControl) updatePetSet(
 	}
 
 	// Fix pod claims for condemned pods, if necessary.
-	if utilfeature.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
+	if features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
 		fixPodClaim := func(i int) (bool, error) {
 			if matchPolicy, err := ssc.podControl.ClaimsMatchRetentionPolicy(ctx, updateSet, condemned[i]); err != nil {
 				return true, err
@@ -703,7 +702,7 @@ func (ssc *defaultPetSetControl) updatePetSet(
 		return &status, nil
 	}
 
-	if utilfeature.DefaultFeatureGate.Enabled(features.MaxUnavailablePetSet) {
+	if features.DefaultFeatureGate.Enabled(features.MaxUnavailablePetSet) {
 		return updatePetSetAfterInvariantEstablished(ctx,
 			ssc,
 			set,

pkg/controller/petset/pet_set_control_test.go

@@ -236,7 +236,7 @@
 		t.Error("Failed to set UpdatedReplicas correctly")
 	}
 	// Check all pods have correct pod index label.
-	if utilfeature.DefaultFeatureGate.Enabled(features.PodIndexLabel) {
+	if features.DefaultFeatureGate.Enabled(features.PodIndexLabel) {
 		selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
 		if err != nil {
 			t.Error(err)
@@ -2713,7 +2713,7 @@
 	return om.podsLister.Pods(set.Namespace).List(selector)
 }
 
 var _ StatefulPodControlObjectManager = &fakeObjectManager{}
 
 type fakeStatefulSetStatusUpdater struct {
 	setsLister          apilisters.PetSetLister
@@ -2871,7 +2871,7 @@
 		WhenScaled:  apps.RetainPersistentVolumeClaimRetentionPolicyType,
 		WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
 	}
-	if set.Spec.PersistentVolumeClaimRetentionPolicy != nil && utilfeature.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
+	if set.Spec.PersistentVolumeClaimRetentionPolicy != nil && features.DefaultFeatureGate.Enabled(features.PetSetAutoDeletePVC) {
 		policy = *set.Spec.PersistentVolumeClaimRetentionPolicy
 	}
 	claimShouldBeRetained := policy.WhenScaled == apps.RetainPersistentVolumeClaimRetentionPolicyType

pkg/controller/petset/pet_set_utils.go

@@ -34,7 +34,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/klog/v2"
 )
@@ -91,7 +90,7 @@ func getOrdinal(pod *v1.Pod) int {
 // getStartOrdinal gets the first possible ordinal (inclusive).
 // Returns spec.ordinals.start if spec.ordinals is set, otherwise returns 0.
 func getStartOrdinal(set *api.PetSet) int {
-	if utilfeature.DefaultFeatureGate.Enabled(features.PetSetStartOrdinal) {
+	if features.DefaultFeatureGate.Enabled(features.PetSetStartOrdinal) {
 		if set.Spec.Ordinals != nil {
 			return int(set.Spec.Ordinals.Start)
 		}
@@ -399,7 +398,7 @@ func updateIdentity(set *api.PetSet, pod *v1.Pod) {
 		pod.Labels = make(map[string]string)
 	}
 	pod.Labels[apps.StatefulSetPodNameLabel] = pod.Name
-	if utilfeature.DefaultFeatureGate.Enabled(features.PodIndexLabel) {
+	if features.DefaultFeatureGate.Enabled(features.PodIndexLabel) {
 		pod.Labels[apps.PodIndexLabel] = strconv.Itoa(ordinal)
 	}
 }

pkg/features/feature_gate.go

@@ -0,0 +1,33 @@
+/*
+Copyright AppsCode Inc. and Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"k8s.io/component-base/featuregate"
+)
+
+var (
+	// DefaultMutableFeatureGate is a mutable version of DefaultFeatureGate.
+	// Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this.
+	// Tests that need to modify feature gates for the duration of their test should use:
+	//   defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)()
+	DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate()
+
+	// DefaultFeatureGate is a shared global FeatureGate.
+	// Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate.
+	DefaultFeatureGate featuregate.FeatureGate = DefaultMutableFeatureGate
+)