Av 193293 release 1.11.3 (#1337)

* AV-193293: Fix: VSes are getting deleted when there is an issue with an access to the kube_api server resulting in setting DeleteConfig to true (#1335)

* Doc changes for 1.11.3

CHANGELOG.md

@@ -429,11 +429,16 @@ All notable changes to this project will be documented in this file. The format
  - Fix: Dedicated insecure VS gets both port `80` and `443` by default when no listeners are defined via any CRD.
  - Fix: If only `loadBalancerIP` is given and no `listeners` are specified in the HostRule `tcpSettings` section, the VS creation fails with `error: Service mandatory for Virtual Service`.
 
- ### Known Issues
+### Known Issues
  - For Avi Controller version less than `30.1.1`, when using SSORule CRD, if `authnReqAcsType` is set to `SAML_AUTHN_REQ_ACS_TYPE_URL`, **Request Body Buffering** needs to be enabled for EVH parent virtual service for SAML traffic to work. This can be achieved by creating a custom application profile of type `HTTP` in the AVI controller with **Request Body Buffering** enabled and attaching that application profile to the EVH parent VS by using a HostRule CRD object. Avi Controller version `30.1.1` is supported starting with AKO version `1.11.1`.
 
 ## AKO-1.11.2
 
 ### Fixed
  - Security issues, present in Go lang net/http packages.
 
+## AKO-1.11.3
+
+### Fixed
+ - Fix: Certificate, from hostrule CRD, can not be assigned to L7 VirtualService when AKO is deployed in Dedicated mode.
+ - Fix: Virtualservices are getting deleted when there is an issue with an access to the kube_api server which results in setting up `deleteConfig` flag to true.

ako-gateway-api/k8s/ako_init.go

@@ -416,7 +416,12 @@ func (c *GatewayController) cleanupStaleVSes() {
 	aviRestClientPool := avicache.SharedAVIClients()
 	aviObjCache := avicache.SharedAviObjCache()
 
-	delModels := k8s.DeleteConfigFromConfigmap(c.informers.ClientSet)
+	delModels, err := k8s.DeleteConfigFromConfigmap(c.informers.ClientSet)
+	if err != nil {
+		c.DisableSync = true
+		utils.AviLog.Errorf("Error occurred while fetching values from configmap. Err: %s", utils.Stringify(err))
+		return
+	}
 	if delModels {
 		go k8s.SetDeleteSyncChannel()
 		parentKeys := aviObjCache.VsCacheMeta.AviCacheGetAllParentVSKeys()

buildsettings.json

@@ -1,5 +1,5 @@
 {
-    "version": "1.11.2",
+    "version": "1.11.3",
     "avi": {
         "maxVersion": "22.1.4",
         "minVersion": "21.1.5"

docs/ako_ha.md

@@ -37,7 +37,7 @@ Passive AKO does the following:
 * Change the `replicaCount` in `values.yaml` to two.
 * Execute the helm upgrade command and provide the updated `values.yaml` file.
 
-helm upgrade ako-1593523840 oci://projects.registry.vmware.com/ako/helm-charts/ako -f /path/to/values.yaml --version 1.11.2 --set ControllerSettings.controllerHost=<IP or Hostname> --set avicredentials.password=<username> --set avicredentials.username=<username> --namespace=avi-system
+helm upgrade ako-1593523840 oci://projects.registry.vmware.com/ako/helm-charts/ako -f /path/to/values.yaml --version 1.11.3 --set ControllerSettings.controllerHost=<IP or Hostname> --set avicredentials.password=<username> --set avicredentials.username=<username> --namespace=avi-system
 
 **Note:**
 1. Currently, more than two replicas are not supported.

docs/install/helm.md

@@ -14,22 +14,22 @@ kubectl create ns avi-system
 Step 2: Search the available charts for AKO
 
 ```
-helm show chart oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.2
+helm show chart oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.3
 
-Pulled: projects.registry.vmware.com/ako/helm-charts/ako:1.11.2
+Pulled: projects.registry.vmware.com/ako/helm-charts/ako:1.11.3
 Digest: sha256:xyxyxxyxyx
 apiVersion: v2
-appVersion: 1.11.2
+appVersion: 1.11.3
 description: A helm chart for Avi Kubernetes Operator
 name: ako
 type: application
-version: 1.11.2
+version: 1.11.3
 ```
 
 Use the `values.yaml` from this chart to edit values related to Avi configuration. To get the values.yaml for a release, run the following command
 
 ```
-helm show values oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.2 > values.yaml
+helm show values oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.3 > values.yaml
 
 ```
 
@@ -44,12 +44,12 @@ Starting AKO-1.7.1, multiple AKO instances can be installed in a cluster.
 
 <b>Primary AKO installation</b>
 ```
-helm install --generate-name oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.2 -f /path/to/values.yaml  --set ControllerSettings.controllerHost=<controller IP or Hostname> --set avicredentials.username=<avi-ctrl-username> --set avicredentials.password=<avi-ctrl-password> --set AKOSettings.primaryInstance=true --namespace=avi-system
+helm install --generate-name oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.3 -f /path/to/values.yaml  --set ControllerSettings.controllerHost=<controller IP or Hostname> --set avicredentials.username=<avi-ctrl-username> --set avicredentials.password=<avi-ctrl-password> --set AKOSettings.primaryInstance=true --namespace=avi-system
 ```
 
 <b>Secondary AKO installation</b>
 ```
-helm install --generate-name oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.2 -f /path/to/values.yaml  --set ControllerSettings.controllerHost=<controller IP or Hostname> --set avicredentials.username=<avi-ctrl-username> --set avicredentials.password=<avi-ctrl-password> --set AKOSettings.primaryInstance=false --namespace=avi-system
+helm install --generate-name oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.3 -f /path/to/values.yaml  --set ControllerSettings.controllerHost=<controller IP or Hostname> --set avicredentials.username=<avi-ctrl-username> --set avicredentials.password=<avi-ctrl-password> --set AKOSettings.primaryInstance=false --namespace=avi-system
 
 ```
 
@@ -59,7 +59,7 @@ Step 4: Check the installation
 helm list -n avi-system
 
 NAME          	NAMESPACE 	REVISION	UPDATED     STATUS  	CHART    	APP VERSION
-ako-1691752136	avi-system	1       	2023-09-28	deployed	ako-1.11.2	1.11.2
+ako-1691752136	avi-system	1       	2023-09-28	deployed	ako-1.11.3	1.11.3
 ```
 
 ## Uninstall using *helm*
@@ -89,7 +89,7 @@ Follow these steps if you are upgrading from an older AKO release.
 Helm does not upgrade the CRDs during a release upgrade. Before you upgrade a release, run the following command to download and upgrade the CRDs:
 
 ```
-helm template oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.2 --include-crds --output-dir <output_dir>
+helm template oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.3 --include-crds --output-dir <output_dir>
 ```
 
 This will save the helm files to an output directory which will contain the CRDs corresponding to the AKO version.
@@ -110,18 +110,18 @@ ako-1593523840	avi-system	1       	2023-04-16 13:44:31.609195757 +0000 UTC	    d
 
 *Step3*
 
-Get the values.yaml for the AKO version 1.11.2 and edit the values as per the requirement.
+Get the values.yaml for the AKO version 1.11.3 and edit the values as per the requirement.
 
 ```
-helm show values oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.2 > values.yaml
+helm show values oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.3 > values.yaml
 
 ```
 *Step4*
 
 Upgrade the helm chart
 
 ```
-helm upgrade ako-1593523840  oci://projects.registry.vmware.com/ako/helm-charts/ako -f /path/to/values.yaml --version 1.11.2 --set ControllerSettings.controllerHost=<IP or Hostname> --set avicredentials.password=<username> --set avicredentials.username=<username> --namespace=avi-system
+helm upgrade ako-1593523840  oci://projects.registry.vmware.com/ako/helm-charts/ako -f /path/to/values.yaml --version 1.11.3 --set ControllerSettings.controllerHost=<IP or Hostname> --set avicredentials.password=<username> --set avicredentials.username=<username> --namespace=avi-system
 
 ```
 

docs/openshift/openshift_helm.md

@@ -17,24 +17,24 @@ oc new-project avi-system
 Search for available charts
 
 ```
-helm show chart oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.2
+helm show chart oci://projects.registry.vmware.com/ako/helm-charts/ako --version 3
 
-Pulled: projects.registry.vmware.com/ako/helm-charts/ako:1.11.2
+Pulled: projects.registry.vmware.com/ako/helm-charts/ako:1.11.3
 Digest: sha256:xyxyxxyxyx
 apiVersion: v2
-appVersion: 1.11.2
+appVersion: 1.11.3
 description: A helm chart for Avi Kubernetes Operator
 name: ako
 type: application
-version: 1.11.2
+version: 1.11.3
 ```
 
 *Step-3*
 
 Edit the [values.yaml](../install/helm.md#parameters) file and update the details according to your environment.
 
 ```
-helm show values oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.2 > values.yaml
+helm show values oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.3 > values.yaml
 
 ```
 
@@ -43,7 +43,7 @@ helm show values oci://projects.registry.vmware.com/ako/helm-charts/ako --versio
 Install AKO.
 
 ```
-helm install --generate-name oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.2 -f /path/to/values.yaml  --set ControllerSettings.controllerHost=<controller IP or Hostname> --set avicredentials.username=<avi-ctrl-username> --set avicredentials.password=<avi-ctrl-password> --namespace=avi-system
+helm install --generate-name oci://projects.registry.vmware.com/ako/helm-charts/ako --version 1.11.3 -f /path/to/values.yaml  --set ControllerSettings.controllerHost=<controller IP or Hostname> --set avicredentials.username=<avi-ctrl-username> --set avicredentials.password=<avi-ctrl-password> --namespace=avi-system
 ```
 
 
@@ -55,7 +55,7 @@ Verify the installation
 helm list -n avi-system
 
 NAME          	NAMESPACE 	REVISION	UPDATED     STATUS  	CHART    	APP VERSION
-ako-1691752136	avi-system	1       	2023-09-28	deployed	ako-1.11.2	1.11.2
+ako-1691752136	avi-system	1       	2023-09-28	deployed	ako-1.11.3	1.11.3
 ```
 
 

internal/k8s/ako_init.go

@@ -69,12 +69,16 @@ func PopulateCache() error {
 	return nil
 }
 
-func (c *AviController) cleanupStaleVSes() {
+func (c *AviController) CleanupStaleVSes() {
 
 	aviRestClientPool := avicache.SharedAVIClients()
 	aviObjCache := avicache.SharedAviObjCache()
-
-	delModels := DeleteConfigFromConfigmap(c.informers.ClientSet)
+	delModels, err := DeleteConfigFromConfigmap(c.informers.ClientSet)
+	if err != nil {
+		c.DisableSync = true
+		utils.AviLog.Errorf("Error occurred while fetching values from configmap. Err: %s", utils.Stringify(err))
+		return
+	}
 	if delModels {
 		go SetDeleteSyncChannel()
 		parentKeys := aviObjCache.VsCacheMeta.AviCacheGetAllParentVSKeys()
@@ -162,14 +166,14 @@ func delConfigFromData(data map[string]string) bool {
 	return delConf
 }
 
-func DeleteConfigFromConfigmap(cs kubernetes.Interface) bool {
+func DeleteConfigFromConfigmap(cs kubernetes.Interface) (bool, error) {
 	cmNS := utils.GetAKONamespace()
 	cm, err := cs.CoreV1().ConfigMaps(cmNS).Get(context.TODO(), lib.AviConfigMap, metav1.GetOptions{})
 	if err == nil {
-		return delConfigFromData(cm.Data)
+		return delConfigFromData(cm.Data), err
 	}
 	utils.AviLog.Warnf("error while reading configmap, sync would be disabled: %v", err)
-	return true
+	return true, err
 }
 
 func (c *AviController) SetSEGroupCloudNameFromNSAnnotations() bool {
@@ -257,7 +261,7 @@ func (c *AviController) HandleConfigMap(k8sinfo K8sinformers, ctrlCh chan struct
 	if !validateUserInput {
 		return errors.New("sync is disabled because of configmap unavailability during bootup")
 	}
-	c.DisableSync = DeleteConfigFromConfigmap(cs)
+	c.DisableSync, _ = DeleteConfigFromConfigmap(cs)
 	lib.SetDisableSync(c.DisableSync)
 
 	configMapEventHandler := cache.ResourceEventHandlerFuncs{

internal/k8s/leader_election_callbacks.go

@@ -56,7 +56,7 @@ func (c *AviController) OnStoppedLeading() {
 // Event Handler functions
 func (c *AviController) OnStartedLeadingDuringBootup() {
 	c.publishAllParentVSKeysToRestLayer()
-	c.cleanupStaleVSes()
+	c.CleanupStaleVSes()
 	// once the l3 cache is populated, we can call the updatestatus functions from here
 	restlayer := rest.NewRestOperations(avicache.SharedAviObjCache(), avicache.SharedAVIClients())
 	restlayer.SyncObjectStatuses()
@@ -72,7 +72,7 @@ func (c *AviController) OnStartedLeadingAfterFailover() {
 
 func (c *AviController) OnNewLeaderDuringBootup() {
 	c.publishAllParentVSKeysToRestLayer()
-	c.cleanupStaleVSes()
+	c.CleanupStaleVSes()
 }
 
 func (c *AviController) OnLostLeadership() {

tests/bootuptests/stale_obj_delete_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/cache"
 	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/k8s"
@@ -24,6 +25,7 @@ import (
 var KubeClient *k8sfake.Clientset
 var CRDClient *crdfake.Clientset
 var V1beta1CRDClient *v1beta1crdfake.Clientset
+var ctrl *k8s.AviController
 var restChan chan bool
 var uuidMap map[string]bool
 
@@ -83,7 +85,7 @@ func TestMain(m *testing.M) {
 
 	integrationtest.NewAviFakeClientInstance(KubeClient, true)
 	defer integrationtest.AviFakeClientInstance.Close()
-
+	ctrl = k8s.SharedAviController()
 	os.Exit(m.Run())
 }
 
@@ -166,3 +168,15 @@ func TestNetworkIssueCacheValidationDuringBootup(t *testing.T) {
 	}
 	integrationtest.ResetMiddleware()
 }
+
+func TestConfigmapDeletion(t *testing.T) {
+	integrationtest.AddConfigMap(KubeClient)
+	time.Sleep(10 * time.Second)
+	integrationtest.DeleteConfigMap(KubeClient, t)
+	ctrl.CleanupStaleVSes()
+	// Simulated error condition while fetching configmap by deleting it.
+	// if Disablesync is false or DeleteConfig is true, fail the test case.
+	if !ctrl.DisableSync || lib.GetDeleteConfigMap() {
+		t.Fatalf("Validation for cofigmapDelete Failed.")
+	}
+}

tests/integrationtest/lib.go

@@ -112,6 +112,14 @@ func AddConfigMap(client *k8sfake.Clientset) {
 	client.CoreV1().ConfigMaps(utils.GetAKONamespace()).Create(context.TODO(), aviCM, metav1.CreateOptions{})
 }
 
+func DeleteConfigMap(kubeClient *k8sfake.Clientset, t *testing.T) {
+	err := kubeClient.CoreV1().ConfigMaps(utils.GetAKONamespace()).Delete(context.TODO(), "avi-k8s-config", metav1.DeleteOptions{})
+	if err != nil {
+		t.Fatalf("error in deleting configmap: %v", err)
+	}
+	time.Sleep(10 * time.Second)
+}
+
 func AddDefaultIngressClass() {
 	aviIngressClass := &networking.IngressClass{
 		ObjectMeta: metav1.ObjectMeta{

version.yaml

@@ -1,4 +1,4 @@
 major: 1
 minor: 11
-maintenance: 2
+maintenance: 3
 patch: null