AV-193293: Fix: VSes are getting deleted when there is an issue with …

…an access to the kube_api server resulting in setting DeleteConfig to true (vmware#1335)

ako-gateway-api/k8s/ako_init.go

@@ -416,7 +416,12 @@ func (c *GatewayController) cleanupStaleVSes() {
 	aviRestClientPool := avicache.SharedAVIClients()
 	aviObjCache := avicache.SharedAviObjCache()
 
-	delModels := k8s.DeleteConfigFromConfigmap(c.informers.ClientSet)
+	delModels, err := k8s.DeleteConfigFromConfigmap(c.informers.ClientSet)
+	if err != nil {
+		c.DisableSync = true
+		utils.AviLog.Errorf("Error occurred while fetching values from configmap. Err: %s", utils.Stringify(err))
+		return
+	}
 	if delModels {
 		go k8s.SetDeleteSyncChannel()
 		parentKeys := aviObjCache.VsCacheMeta.AviCacheGetAllParentVSKeys()

internal/k8s/ako_init.go

@@ -69,12 +69,16 @@ func PopulateCache() error {
 	return nil
 }
 
-func (c *AviController) cleanupStaleVSes() {
+func (c *AviController) CleanupStaleVSes() {
 
 	aviRestClientPool := avicache.SharedAVIClients()
 	aviObjCache := avicache.SharedAviObjCache()
-
-	delModels := DeleteConfigFromConfigmap(c.informers.ClientSet)
+	delModels, err := DeleteConfigFromConfigmap(c.informers.ClientSet)
+	if err != nil {
+		c.DisableSync = true
+		utils.AviLog.Errorf("Error occurred while fetching values from configmap. Err: %s", utils.Stringify(err))
+		return
+	}
 	if delModels {
 		go SetDeleteSyncChannel()
 		parentKeys := aviObjCache.VsCacheMeta.AviCacheGetAllParentVSKeys()
@@ -162,14 +166,14 @@ func delConfigFromData(data map[string]string) bool {
 	return delConf
 }
 
-func DeleteConfigFromConfigmap(cs kubernetes.Interface) bool {
+func DeleteConfigFromConfigmap(cs kubernetes.Interface) (bool, error) {
 	cmNS := utils.GetAKONamespace()
 	cm, err := cs.CoreV1().ConfigMaps(cmNS).Get(context.TODO(), lib.AviConfigMap, metav1.GetOptions{})
 	if err == nil {
-		return delConfigFromData(cm.Data)
+		return delConfigFromData(cm.Data), err
 	}
 	utils.AviLog.Warnf("error while reading configmap, sync would be disabled: %v", err)
-	return true
+	return true, err
 }
 
 func (c *AviController) SetSEGroupCloudNameFromNSAnnotations() bool {
@@ -257,7 +261,7 @@ func (c *AviController) HandleConfigMap(k8sinfo K8sinformers, ctrlCh chan struct
 	if !validateUserInput {
 		return errors.New("sync is disabled because of configmap unavailability during bootup")
 	}
-	c.DisableSync = DeleteConfigFromConfigmap(cs)
+	c.DisableSync, _ = DeleteConfigFromConfigmap(cs)
 	lib.SetDisableSync(c.DisableSync)
 
 	configMapEventHandler := cache.ResourceEventHandlerFuncs{

internal/k8s/leader_election_callbacks.go

@@ -56,7 +56,7 @@ func (c *AviController) OnStoppedLeading() {
 // Event Handler functions
 func (c *AviController) OnStartedLeadingDuringBootup() {
 	c.publishAllParentVSKeysToRestLayer()
-	c.cleanupStaleVSes()
+	c.CleanupStaleVSes()
 	// once the l3 cache is populated, we can call the updatestatus functions from here
 	restlayer := rest.NewRestOperations(avicache.SharedAviObjCache(), avicache.SharedAVIClients())
 	restlayer.SyncObjectStatuses()
@@ -72,7 +72,7 @@ func (c *AviController) OnStartedLeadingAfterFailover() {
 
 func (c *AviController) OnNewLeaderDuringBootup() {
 	c.publishAllParentVSKeysToRestLayer()
-	c.cleanupStaleVSes()
+	c.CleanupStaleVSes()
 }
 
 func (c *AviController) OnLostLeadership() {

tests/bootuptests/stale_obj_delete_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/cache"
 	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/k8s"
@@ -24,6 +25,7 @@ import (
 var KubeClient *k8sfake.Clientset
 var CRDClient *crdfake.Clientset
 var V1beta1CRDClient *v1beta1crdfake.Clientset
+var ctrl *k8s.AviController
 var restChan chan bool
 var uuidMap map[string]bool
 
@@ -83,7 +85,7 @@ func TestMain(m *testing.M) {
 
 	integrationtest.NewAviFakeClientInstance(KubeClient, true)
 	defer integrationtest.AviFakeClientInstance.Close()
-
+	ctrl = k8s.SharedAviController()
 	os.Exit(m.Run())
 }
 
@@ -166,3 +168,15 @@ func TestNetworkIssueCacheValidationDuringBootup(t *testing.T) {
 	}
 	integrationtest.ResetMiddleware()
 }
+
+func TestConfigmapDeletion(t *testing.T) {
+	integrationtest.AddConfigMap(KubeClient)
+	time.Sleep(10 * time.Second)
+	integrationtest.DeleteConfigMap(KubeClient, t)
+	ctrl.CleanupStaleVSes()
+	// Simulated error condition while fetching configmap by deleting it.
+	// if Disablesync is false or DeleteConfig is true, fail the test case.
+	if !ctrl.DisableSync || lib.GetDeleteConfigMap() {
+		t.Fatalf("Validation for cofigmapDelete Failed.")
+	}
+}

tests/integrationtest/lib.go

@@ -112,6 +112,14 @@ func AddConfigMap(client *k8sfake.Clientset) {
 	client.CoreV1().ConfigMaps(utils.GetAKONamespace()).Create(context.TODO(), aviCM, metav1.CreateOptions{})
 }
 
+func DeleteConfigMap(kubeClient *k8sfake.Clientset, t *testing.T) {
+	err := kubeClient.CoreV1().ConfigMaps(utils.GetAKONamespace()).Delete(context.TODO(), "avi-k8s-config", metav1.DeleteOptions{})
+	if err != nil {
+		t.Fatalf("error in deleting configmap: %v", err)
+	}
+	time.Sleep(10 * time.Second)
+}
+
 func AddDefaultIngressClass() {
 	aviIngressClass := &networking.IngressClass{
 		ObjectMeta: metav1.ObjectMeta{

version.yaml

@@ -1,4 +1,4 @@
 major: 1
 minor: 11
-maintenance: 2
+maintenance: 3
 patch: null