diff --git a/cmd/ako-main/main.go b/cmd/ako-main/main.go index b6c3e5164..d74bdb6ea 100644 --- a/cmd/ako-main/main.go +++ b/cmd/ako-main/main.go @@ -111,6 +111,7 @@ func InitializeAKC() { akoControlConfig.SetAKOBlockedNSList(lib.GetGlobalBlockedNSList()) akoControlConfig.SetControllerVRFContext(lib.GetControllerVRFContext()) akoControlConfig.SetAKOPrometheusFlag(lib.IsPrometheusEnabled()) + akoControlConfig.SetAKOFQDNReusePolicy(strings.ToLower(os.Getenv("FQDN_REUSE_POLICY"))) var crdClient *crd.Clientset var advl4Client *advl4.Clientset diff --git a/helm/ako/templates/configmap.yaml b/helm/ako/templates/configmap.yaml index c22bf5010..a99fbfca9 100644 --- a/helm/ako/templates/configmap.yaml +++ b/helm/ako/templates/configmap.yaml @@ -53,3 +53,4 @@ data: useDefaultSecretsOnly: {{ .Values.AKOSettings.useDefaultSecretsOnly | quote }} enablePrometheus: {{ default "false" .Values.featureGates.EnablePrometheus | quote }} enableEndpointSlice: {{ default "false" .Values.featureGates.EnableEndpointSlice | quote }} + fqdnReusePolicy: {{ default "InterNamespaceAllowed" .Values.L7Settings.fqdnReusePolicy | quote}} diff --git a/helm/ako/templates/statefulset.yaml b/helm/ako/templates/statefulset.yaml index 56fc815d7..0298e61a0 100644 --- a/helm/ako/templates/statefulset.yaml +++ b/helm/ako/templates/statefulset.yaml @@ -288,6 +288,11 @@ spec: configMapKeyRef: name: avi-k8s-config key: enableEndpointSlice + - name: FQDN_REUSE_POLICY + valueFrom: + configMapKeyRef: + name: avi-k8s-config + key: fqdnReusePolicy resources: {{- toYaml .Values.resources | nindent 12 }} livenessProbe: diff --git a/helm/ako/values.yaml b/helm/ako/values.yaml index 5834d6495..e73048f95 100644 --- a/helm/ako/values.yaml +++ b/helm/ako/values.yaml @@ -92,6 +92,7 @@ L7Settings: shardVSSize: "LARGE" # Use this to control the layer 7 VS numbers. This applies to both secure/insecure VSes but does not apply for passthrough. ENUMs: LARGE, MEDIUM, SMALL, DEDICATED passthroughShardSize: "SMALL" # Control the passthrough virtualservice numbers using this ENUM. ENUMs: LARGE, MEDIUM, SMALL enableMCI: "false" # Enabling this flag would tell AKO to start processing multi-cluster ingress objects. + fqdnReusePolicy: "InterNamespaceAllowed" # Use this to control whether AKO allows cross-namespace usage of FQDNs. enum Strict|InterNamespaceAllowed ### This section outlines all the knobs used to control Layer 4 loadbalancing settings in AKO. L4Settings: diff --git a/internal/cache/controller_obj_cache.go b/internal/cache/controller_obj_cache.go index 764d4c51c..a198aad40 100644 --- a/internal/cache/controller_obj_cache.go +++ b/internal/cache/controller_obj_cache.go @@ -27,6 +27,7 @@ import ( "sync" "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/lib" + "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/objects" akov1beta1 "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/apis/ako/v1beta1" pq "github.com/jupp0r/go-priority-queue" @@ -2186,8 +2187,17 @@ func (c *AviObjCache) AviObjVSCachePopulate(client *clients.AviClient, cloud str if err := json.Unmarshal([]byte(svc_mdata_intf.(string)), &svc_mdata_obj); err != nil { utils.AviLog.Warnf("Error parsing service metadata during vs cache :%v", err) + } else if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + // call this only when FQDN policy is strict + hostToIngMapping := svc_mdata_obj.HostToNamespaceIngressName + utils.AviLog.Debugf("HosttoIng mapping is %v", utils.Stringify(hostToIngMapping)) + if hostToIngMapping != nil { + // Now populate the map + PopulateHostToIngMapping(hostToIngMapping) + } } } + var sni_child_collection []string vh_child, found := vs["vh_child_vs_uuid"] if found { @@ -2397,7 +2407,6 @@ func (c *AviObjCache) AviObjVSCachePopulate(client *clients.AviClient, cloud str } c.VsCacheLocal.AviCacheAdd(k, &vsMetaObj) utils.AviLog.Debugf("Added VS cache key :%s", utils.Stringify(&vsMetaObj)) - } } if resp["next"] != nil { @@ -2415,6 +2424,48 @@ func (c *AviObjCache) AviObjVSCachePopulate(client *clients.AviClient, cloud str return nil } +// Upfront populate mapping so that during FullSyncK8s, it will be used to assign ingresses/routes to appropriate hosts list. +func PopulateHostToIngMapping(hostsToIng map[string][]string) { + isRoute := false + if utils.GetInformers().RouteInformer != nil { + isRoute = true + } + var routeNamespaceName objects.RouteNamspaceName + for host, ings := range hostsToIng { + // append each ingress in active list + utils.AviLog.Debugf("Populating Ingress mapping for host : %s", host) + for _, ing := range ings { + namespace, _, name := lib.ExtractTypeNameNamespace(ing) + // Fetch ingress using clientset. From informer couldn't fetch it. + if !isRoute { + ingObj, err := utils.GetInformers().IngressInformer.Lister().Ingresses(namespace).Get(name) + if err != nil { + utils.AviLog.Errorf("Unable to retrieve the ingress %s/%s during populating host to ingress map in populate cache: %s", namespace, name, err) + continue + } + routeNamespaceName = objects.RouteNamspaceName{ + RouteNSRouteName: utils.Ingress + "/" + ing, + CreationTime: ingObj.CreationTimestamp, + } + } else { + routeObj, err := utils.GetInformers().RouteInformer.Lister().Routes(namespace).Get(name) + if err != nil { + utils.AviLog.Errorf("Unable to retrieve the ingress %s/%s during populating host to ingress map in populate cache: %s", namespace, name, err) + continue + } + routeNamespaceName = objects.RouteNamspaceName{ + RouteNSRouteName: utils.OshiftRoute + "/" + ing, + CreationTime: routeObj.CreationTimestamp, + } + } + + // Add it to the structure + objects.SharedUniqueNamespaceLister().UpdateHostnameToRoute(host, routeNamespaceName) + + } + } +} + func (c *AviObjCache) AviObjOneVSCachePopulate(client *clients.AviClient, cloud string, vsName, tenant string) error { // This method should be called only from layer-3 during a retry. var rest_response interface{} @@ -2458,6 +2509,14 @@ func (c *AviObjCache) AviObjOneVSCachePopulate(client *clients.AviClient, cloud if err := json.Unmarshal([]byte(svc_mdata_intf.(string)), &svc_mdata_obj); err != nil { utils.AviLog.Warnf("Error parsing service metadata during vs cache :%v", err) + } else if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + // call this only when FQDN policy is strict + hostToIngMapping := svc_mdata_obj.HostToNamespaceIngressName + utils.AviLog.Debugf("HosttoIng mapping is %v", utils.Stringify(hostToIngMapping)) + if hostToIngMapping != nil { + // Now populate the map + PopulateHostToIngMapping(hostToIngMapping) + } } } var sni_child_collection []string diff --git a/internal/k8s/ako_init.go b/internal/k8s/ako_init.go index 0a8eb8892..d38f1d65a 100644 --- a/internal/k8s/ako_init.go +++ b/internal/k8s/ako_init.go @@ -530,6 +530,8 @@ func (c *AviController) InitController(informers K8sinformers, registeredInforme statusQueueParams := utils.WorkerQueue{NumWorkers: numGraphWorkers, WorkqueueName: utils.StatusQueue} graphQueue = utils.SharedWorkQueue(&ingestionQueueParams, &graphQueueParams, &slowRetryQParams, &fastRetryQParams, &statusQueueParams).GetQueueByName(utils.GraphLayer) + c.addIndexers() + c.Start(stopCh) err := PopulateCache() if err != nil { c.DisableSync = true @@ -537,9 +539,6 @@ func (c *AviController) InitController(informers K8sinformers, registeredInforme lib.ShutdownApi() } - c.addIndexers() - c.Start(stopCh) - fullSyncInterval := os.Getenv(utils.FULL_SYNC_INTERVAL) interval, err := strconv.ParseInt(fullSyncInterval, 10, 64) @@ -1100,36 +1099,69 @@ func (c *AviController) FullSyncK8s(sync bool) error { } //Ingress Section if utils.GetInformers().IngressInformer != nil { + ingObjList := make([]*networkingv1.Ingress, 0) + // create list of ingresses. for namespace := range acceptedNamespaces { ingObjs, err := utils.GetInformers().IngressInformer.Lister().Ingresses(namespace).List(labels.Set(nil).AsSelector()) if err != nil { utils.AviLog.Errorf("Unable to retrieve the ingresses during full sync: %s", err) - } else { - for _, ingObj := range ingObjs { - key := utils.Ingress + "/" + utils.ObjKey(ingObj) - meta, err := meta.Accessor(ingObj) - if err == nil { - resVer := meta.GetResourceVersion() - objects.SharedResourceVerInstanceLister().Save(key, resVer) - } - utils.AviLog.Debugf("Dequeue for ingress key: %v", key) - lib.IncrementQueueCounter(utils.ObjectIngestionLayer) - nodes.DequeueIngestion(key, true) + continue + } + ingObjList = append(ingObjList, ingObjs...) + } + // sort the list as per the timestamp. Sorting logic can be kept irrespective of strict or non-strict policy + sort.Slice(ingObjList, func(i, j int) bool { return lib.IngressLessthan(ingObjList[i], ingObjList[j]) }) + + for _, ingObj := range ingObjList { + key := utils.Ingress + "/" + ingObj.Namespace + "/" + ingObj.Name + isValid := true + + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + // get the hostnames in the ingress + isValid, _ = isIngAcceptedWithFQDNRestriction(key, ingObj) + } + if isValid { + meta, err := meta.Accessor(ingObj) + if err == nil { + resVer := meta.GetResourceVersion() + objects.SharedResourceVerInstanceLister().Save(key, resVer) } + utils.AviLog.Debugf("Dequeue for ingress key: %v", key) + lib.IncrementQueueCounter(utils.ObjectIngestionLayer) + nodes.DequeueIngestion(key, true) + } else { + utils.AviLog.Warnf("key: %s, msg: Ingress is not accepted due to FQDN restriction policy", key) } } + // TODO: free ingObjList } //Route Section if utils.GetInformers().RouteInformer != nil { - routeObjs, err := utils.GetInformers().RouteInformer.Lister().List(labels.Set(nil).AsSelector()) - if err != nil { - utils.AviLog.Errorf("Unable to retrieve the routes during full sync: %s", err) - } else { - for _, routeObj := range routeObjs { - if _, ok := acceptedNamespaces[routeObj.Namespace]; !ok { - continue + routeObjList := make([]*routev1.Route, 0) + for namespace := range acceptedNamespaces { + routeObjs, err := utils.GetInformers().RouteInformer.Lister().Routes(namespace).List(labels.Set(nil).AsSelector()) + if err != nil { + utils.AviLog.Errorf("Unable to retrieve the ingresses during full sync: %s", err) + continue + } + routeObjList = append(routeObjList, routeObjs...) + } + + // sort on timestamp + sort.Slice(routeObjList, func(i, j int) bool { return lib.RouteLessthan(routeObjList[i], routeObjList[j]) }) + + for _, routeObj := range routeObjList { + key := utils.OshiftRoute + "/" + utils.ObjKey(routeObj) + isValid := true + + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + isValid = isRouteAcceptedWithFQDNRestriction(key, routeObj) + if isValid { + utils.AviLog.Debugf("Route %s is added to active list. Enqueuing it", key) } - key := utils.OshiftRoute + "/" + utils.ObjKey(routeObj) + } + if isValid { + // Enqueue it only in case of valid meta, err := meta.Accessor(routeObj) if err == nil { resVer := meta.GetResourceVersion() @@ -1138,8 +1170,11 @@ func (c *AviController) FullSyncK8s(sync bool) error { utils.AviLog.Debugf("Dequeue for route key: %v", key) lib.IncrementQueueCounter(utils.ObjectIngestionLayer) nodes.DequeueIngestion(key, true) + } else { + utils.AviLog.Warnf("key: %s, msg: Route is not accepted due to FQDN restriction policy", key) } } + // TODO: Free routelist } if lib.UseServicesAPI() { gatewayObjs, err := lib.AKOControlConfig().SvcAPIInformers().GatewayInformer.Lister().Gateways(metav1.NamespaceAll).List(labels.Set(nil).AsSelector()) diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 204d34fb4..2df8b3405 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -198,9 +199,15 @@ func AddIngressFromNSToIngestionQueue(numWorkers uint32, c *AviController, names for _, ingObj := range ingObjs { key := utils.Ingress + "/" + utils.ObjKey(ingObj) bkt := utils.Bkt(namespace, numWorkers) - c.workqueue[bkt].AddRateLimited(key) - lib.IncrementQueueCounter(utils.ObjectIngestionLayer) - utils.AviLog.Debugf("key: %s, msg: %s for namespace: %s", key, msg, namespace) + toBeAdded := true + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + toBeAdded, _ = isIngAcceptedWithFQDNRestriction(key, ingObj) + } + if toBeAdded { + c.workqueue[bkt].AddRateLimited(key) + lib.IncrementQueueCounter(utils.ObjectIngestionLayer) + utils.AviLog.Debugf("key: %s, msg: %s for namespace: %s", key, msg, namespace) + } } } @@ -213,9 +220,15 @@ func AddRoutesFromNSToIngestionQueue(numWorkers uint32, c *AviController, namesp for _, routeObj := range routeObjs { key := utils.OshiftRoute + "/" + utils.ObjKey(routeObj) bkt := utils.Bkt(namespace, numWorkers) - c.workqueue[bkt].AddRateLimited(key) - lib.IncrementQueueCounter(utils.ObjectIngestionLayer) - utils.AviLog.Debugf("key: %s, msg: %s for namespace: %s", key, msg, namespace) + toBeAdded := true + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + toBeAdded = isRouteAcceptedWithFQDNRestriction(key, routeObj) + } + if toBeAdded { + c.workqueue[bkt].AddRateLimited(key) + lib.IncrementQueueCounter(utils.ObjectIngestionLayer) + utils.AviLog.Debugf("key: %s, msg: %s for namespace: %s", key, msg, namespace) + } } } @@ -520,6 +533,13 @@ func AddRouteEventHandler(numWorkers uint32, c *AviController) cache.ResourceEve utils.AviLog.Debugf("key : %s, msg: same resource version returning", key) return } + + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict && !isRouteAcceptedWithFQDNRestriction(key, route) { + // update the status - already host claimed + status.UpdateRouteStatusWithErrMsg(key, route.Name, namespace, lib.HostAlreadyClaimed) + return + } + bkt := utils.Bkt(namespace, numWorkers) if !lib.HasValidBackends(route.Spec, route.Name, namespace, key) { status.UpdateRouteStatusWithErrMsg(key, route.Name, namespace, lib.DuplicateBackends) @@ -527,6 +547,7 @@ func AddRouteEventHandler(numWorkers uint32, c *AviController) cache.ResourceEve c.workqueue[bkt].AddRateLimited(key) lib.IncrementQueueCounter(utils.ObjectIngestionLayer) utils.AviLog.Debugf("key: %s, msg: ADD", key) + }, DeleteFunc: func(obj interface{}) { if c.DisableSync { @@ -551,6 +572,14 @@ func AddRouteEventHandler(numWorkers uint32, c *AviController) cache.ResourceEve utils.AviLog.Debugf("key: %s, msg: Route delete event: Namespace: %s didn't qualify filter. Not deleting route", key, namespace) return } + + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + routeNamespaceName := objects.RouteNamspaceName{ + RouteNSRouteName: key, + CreationTime: route.CreationTimestamp, + } + objects.SharedUniqueNamespaceLister().DeleteHostnameToRoute(route.Spec.Host, routeNamespaceName) + } bkt := utils.Bkt(namespace, numWorkers) c.workqueue[bkt].AddRateLimited(key) lib.IncrementQueueCounter(utils.ObjectIngestionLayer) @@ -570,13 +599,42 @@ func AddRouteEventHandler(numWorkers uint32, c *AviController) cache.ResourceEve utils.AviLog.Debugf("key: %s, msg: Route update event: Namespace: %s didn't qualify filter. Not updating route", key, namespace) return } + bkt := utils.Bkt(namespace, numWorkers) if !lib.HasValidBackends(newRoute.Spec, newRoute.Name, namespace, key) { status.UpdateRouteStatusWithErrMsg(key, newRoute.Name, namespace, lib.DuplicateBackends) } - c.workqueue[bkt].AddRateLimited(key) - lib.IncrementQueueCounter(utils.ObjectIngestionLayer) - utils.AviLog.Debugf("key: %s, msg: UPDATE", key) + if oldRoute.Spec.Host == newRoute.Spec.Host { + // same hosts + isAccepted := isRouteAcceptedWithFQDNRestriction(key, newRoute) + if isAccepted { + c.workqueue[bkt].AddRateLimited(key) + lib.IncrementQueueCounter(utils.ObjectIngestionLayer) + utils.AviLog.Debugf("key: %s, msg: UPDATE", key) + } + } else { + isOldAccepted := isRouteAcceptedWithFQDNRestriction(key, oldRoute) + isNewAccepted := isRouteAcceptedWithFQDNRestriction(key, newRoute) + if !isOldAccepted && !isNewAccepted { + // set status + // update the status - already host claimed + status.UpdateRouteStatusWithErrMsg(key, newRoute.Name, namespace, lib.HostAlreadyClaimed) + return + } + if isOldAccepted { + routeNamespaceName := objects.RouteNamspaceName{ + RouteNSRouteName: key, + CreationTime: oldRoute.CreationTimestamp, + } + // TODO: Recently host field in route has become optional. There is alternate field needs to be used. + // So storing route hostname functionality will undergo changes. + objects.SharedUniqueNamespaceLister().DeleteHostnameToRoute(oldRoute.Spec.Host, routeNamespaceName) + } + c.workqueue[bkt].AddRateLimited(key) + lib.IncrementQueueCounter(utils.ObjectIngestionLayer) + utils.AviLog.Debugf("key: %s, msg: UPDATE", key) + + } } }, } @@ -1251,10 +1309,17 @@ func (c *AviController) SetupEventHandlers(k8sinfo K8sinformers) { if !lib.ValidateIngressForClass(key, ingress) { return } + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + if toBeAdded, _ := isIngAcceptedWithFQDNRestriction(key, ingress); !toBeAdded { + utils.AviLog.Warnf("key: %s, msg: Ingress is not added due to conflict in hostname", key) + return + } + } bkt := utils.Bkt(namespace, numWorkers) c.workqueue[bkt].AddRateLimited(key) lib.IncrementQueueCounter(utils.ObjectIngestionLayer) utils.AviLog.Debugf("key: %s, msg: ADD", key) + }, DeleteFunc: func(obj interface{}) { if c.DisableSync { @@ -1286,10 +1351,18 @@ func (c *AviController) SetupEventHandlers(k8sinfo K8sinformers) { return } objects.SharedResourceVerInstanceLister().Delete(key) + // Add validation here bkt := utils.Bkt(namespace, numWorkers) - c.workqueue[bkt].AddRateLimited(key) + + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + deleteHostnameToRoute(key, ingress) + } + lib.IncrementQueueCounter(utils.ObjectIngestionLayer) utils.AviLog.Debugf("key: %s, msg: DELETE", key) + // This will enqueue key irrespective of model present or not. Can be optimized. + c.workqueue[bkt].AddRateLimited(key) + }, UpdateFunc: func(old, cur interface{}) { if c.DisableSync { @@ -1305,9 +1378,24 @@ func (c *AviController) SetupEventHandlers(k8sinfo K8sinformers) { return } bkt := utils.Bkt(namespace, numWorkers) + + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + oldIngAccepted, oldHosts := isIngAcceptedWithFQDNRestriction(key, oldobj) + newIngAccepted, newHosts := isIngAcceptedWithFQDNRestriction(key, ingress) + if !oldIngAccepted && !newIngAccepted { + utils.AviLog.Warnf("key: %s, msg: Ingress is not added due to conflict in hostname", key) + return + } + + if oldIngAccepted && !oldHosts.Equal(newHosts) { + deleteHostnameToRoute(key, oldobj) + } + } + c.workqueue[bkt].AddRateLimited(key) lib.IncrementQueueCounter(utils.ObjectIngestionLayer) utils.AviLog.Debugf("key: %s, msg: UPDATE", key) + } }, } @@ -1674,3 +1762,59 @@ func (c *AviController) Run(stopCh <-chan struct{}) error { func (c *AviController) GetValidator() Validator { return NewValidator() } +func isIngAcceptedWithFQDNRestriction(key string, ingress *networkingv1.Ingress) (bool, sets.Set[string]) { + routeNamespaceName := objects.RouteNamspaceName{ + RouteNSRouteName: key, + CreationTime: ingress.CreationTimestamp, + } + ingHosts := sets.New[string]() + for _, rule := range ingress.Spec.Rules { + ingHosts.Insert(rule.Host) + } + // Current behaviour if one of fqdn is false, not added. + isAdded := false + for _, host := range sets.List(ingHosts) { + isAdded, _, _ = objects.SharedUniqueNamespaceLister().UpdateHostnameToRoute(host, routeNamespaceName) + if !isAdded { + utils.AviLog.Warnf("key:%s, msg: ingress is not accepted as host %s is already claimed", key, host) + err_msg := fmt.Sprintf("Host %s already claimed", host) + lib.AKOControlConfig().IngressEventf(ingress.ObjectMeta, corev1.EventTypeWarning, lib.IngressUpdateEvent, err_msg) + break + } + } + if !isAdded { + utils.AviLog.Warnf("key: %s, msg: Ingress is not added due to hostname conflict", key) + // Few hosts might have got added, so we need to remove those from the list. + for _, host := range sets.List(ingHosts) { + objects.SharedUniqueNamespaceLister().DeleteHostnameToRoute(host, routeNamespaceName) + } + } + return isAdded, ingHosts +} + +func isRouteAcceptedWithFQDNRestriction(key string, route *routev1.Route) bool { + routeNamespaceName := objects.RouteNamspaceName{ + RouteNSRouteName: key, + CreationTime: route.CreationTimestamp, + } + isAdded := false + isAdded, _, _ = objects.SharedUniqueNamespaceLister().UpdateHostnameToRoute(route.Spec.Host, routeNamespaceName) + if !isAdded { + utils.AviLog.Warnf("key: %s, msg: Route is not added due to hostname conflict", key) + } + return isAdded +} + +func deleteHostnameToRoute(key string, ingress *networkingv1.Ingress) { + routeNamespaceName := objects.RouteNamspaceName{ + RouteNSRouteName: key, + CreationTime: ingress.CreationTimestamp, + } + ingHosts := sets.NewString() + for _, rule := range ingress.Spec.Rules { + ingHosts.Insert(rule.Host) + } + for _, host := range ingHosts.List() { + objects.SharedUniqueNamespaceLister().DeleteHostnameToRoute(host, routeNamespaceName) + } +} diff --git a/internal/lib/constants.go b/internal/lib/constants.go index 6bd462cfb..052cae66d 100644 --- a/internal/lib/constants.go +++ b/internal/lib/constants.go @@ -120,6 +120,7 @@ const ( TLSRoute = "TLSRoute" UDPRoute = "UDPRoute" DuplicateBackends = "MultipleBackendsWithSameServiceError" + HostAlreadyClaimed = "Host already Claimed" DummyVSForStaleData = "DummyVSForStaleData" ControllerReqWaitTime = 300 PassthroughInsecure = "-insecure" @@ -207,6 +208,8 @@ const ( VrfContextObjectNotFoundError = "VrfContext object not found" NetworkNotFoundError = "Network object not found" CtrlVersion_22_1_6 = "22.1.6" + FQDNReusePolicyStrict = "strict" + FQDNReusePolicyOpen = "internamespaceallowed" // AKO Event constants AKOEventComponent = "avi-kubernetes-operator" @@ -228,6 +231,11 @@ const ( AKODeleteConfigDone = "AKODeleteConfigDone" AKODeleteConfigTimeout = "AKODeleteConfigTimeout" AKOGatewayEventComponent = "avi-kubernetes-operator-gateway-api" + IngressAddEvent = "IngressAddEvent" + IngressDeleteEvent = "IngressDeleteEvent" + IngressUpdateEvent = "IngressUpdateEvent" + RouteAddEvent = "RouteAddEvent" + RouteUpdateEvent = "RouteUpdateEvent" DefaultIngressClassAnnotation = "ingressclass.kubernetes.io/is-default-class" ExternalDNSAnnotation = "external-dns.alpha.kubernetes.io/hostname" diff --git a/internal/lib/control_config.go b/internal/lib/control_config.go index 9f66d58de..106df345a 100644 --- a/internal/lib/control_config.go +++ b/internal/lib/control_config.go @@ -22,6 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -164,6 +165,9 @@ type akoControlConfig struct { //endpointSlices Enabled isEndpointSlicesEnabled bool + + //fqdnReusePolicy is set to Strict/InterNamespaceAllowed according to whether AKO allows FQDN sharing across namespaces + fqdnReusePolicy string } var akoControlConfigInstance *akoControlConfig @@ -357,6 +361,27 @@ func (c *akoControlConfig) SetControllerVRFContext(v string) { c.controllerVRFContext = v } +func (c *akoControlConfig) SetAKOFQDNReusePolicy(FQDNPolicy string) { + // Empty or SNI deployment--> Allow across namespace + if FQDNPolicy == "" || !IsEvhEnabled() { + FQDNPolicy = FQDNReusePolicyOpen + } + + if FQDNPolicy != FQDNReusePolicyOpen && FQDNPolicy != FQDNReusePolicyStrict { + // if not one of it, set it to open + FQDNPolicy = FQDNReusePolicyOpen + } + c.fqdnReusePolicy = FQDNPolicy + utils.AviLog.Infof("AKO FQDN reuse policy is: %s", c.fqdnReusePolicy) +} + +// This utility returns FQDN Reuse policy of AKO. +// Strict --> FQDN restrict to one namespace +// InternamespaceAllowed --> FQDN can be spanned across multiple namespaces +func (c *akoControlConfig) GetAKOFQDNReusePolicy() string { + return c.fqdnReusePolicy +} + func initControllerVersion() string { version := os.Getenv("CTRL_VERSION") if version == "" { @@ -436,7 +461,15 @@ func (c *akoControlConfig) PodEventf(eventType, reason, message string, formatAr } } } +func (c *akoControlConfig) IngressEventf(ingMeta metav1.ObjectMeta, eventType, reason, message string, formatArgs ...string) { + + if len(formatArgs) > 0 { + c.EventRecorder().Eventf(&networkingv1.Ingress{ObjectMeta: ingMeta}, eventType, reason, message, formatArgs) + } else { + c.EventRecorder().Event(&networkingv1.Ingress{ObjectMeta: ingMeta}, eventType, reason, message) + } +} func GetResponseFromURI(client *clients.AviClient, uri string) (models.SystemConfiguration, error) { response := models.SystemConfiguration{} err := AviGet(client, uri, &response) diff --git a/internal/lib/lib.go b/internal/lib/lib.go index 46e78b2ec..a5a3e534f 100644 --- a/internal/lib/lib.go +++ b/internal/lib/lib.go @@ -80,19 +80,21 @@ type CRDMetadata struct { } type ServiceMetadataObj struct { - NamespaceIngressName []string `json:"namespace_ingress_name"` - IngressName string `json:"ingress_name"` - Namespace string `json:"namespace"` - HostNames []string `json:"hostnames"` - NamespaceServiceName []string `json:"namespace_svc_name"` // []string{ns/name} - CRDStatus CRDMetadata `json:"crd_status"` - PoolRatio uint32 `json:"pool_ratio"` - PassthroughParentRef string `json:"passthrough_parent_ref"` - PassthroughChildRef string `json:"passthrough_child_ref"` - Gateway string `json:"gateway"` // ns/name - HTTPRoute string `json:"httproute"` // ns/name - InsecureEdgeTermAllow bool `json:"insecureedgetermallow"` - IsMCIIngress bool `json:"is_mci_ingress"` + NamespaceIngressName []string `json:"namespace_ingress_name"` + IngressName string `json:"ingress_name"` + Namespace string `json:"namespace"` + HostNames []string `json:"hostnames"` + NamespaceServiceName []string `json:"namespace_svc_name"` // []string{ns/name} + CRDStatus CRDMetadata `json:"crd_status"` + PoolRatio uint32 `json:"pool_ratio"` + PassthroughParentRef string `json:"passthrough_parent_ref"` + PassthroughChildRef string `json:"passthrough_child_ref"` + Gateway string `json:"gateway"` // ns/name + HTTPRoute string `json:"httproute"` // ns/name + InsecureEdgeTermAllow bool `json:"insecureedgetermallow"` + IsMCIIngress bool `json:"is_mci_ingress"` + FQDNReusePolicy string `json:"fqdn_reuse_policy"` + HostToNamespaceIngressName map[string][]string `json:"host_namespace_ingress_name"` } type ServiceMetadataMappingObjType string diff --git a/internal/lib/utils.go b/internal/lib/utils.go index 16d3b4b9a..f23b21db3 100644 --- a/internal/lib/utils.go +++ b/internal/lib/utils.go @@ -22,7 +22,9 @@ import ( "strings" "sync" + routev1 "github.com/openshift/api/route/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/client-go/tools/cache" "github.com/vmware/alb-sdk/go/clients" @@ -185,7 +187,29 @@ func GetPodsFromService(namespace, serviceName string, targetPortName intstr.Int objects.SharedSvcToPodLister().Save(svcKey, PodsWithTargetPort{Pods: pods, TargetPort: targetPort}) return pods, targetPort } +func IngressLessthan(ing1, ing2 *networkingv1.Ingress) bool { + if ing1.CreationTimestamp.Before(&ing2.CreationTimestamp) { + return true + } + + if ing2.CreationTimestamp.Before(&ing1.CreationTimestamp) { + return false + } + + return ing1.UID < ing2.UID +} +func RouteLessthan(route1, route2 *routev1.Route) bool { + if route1.CreationTimestamp.Before(&route2.CreationTimestamp) { + return true + } + + if route2.CreationTimestamp.Before(&route1.CreationTimestamp) { + return false + } + + return route1.UID < route2.UID +} func GetServicesForPod(pod *corev1.Pod) ([]string, []string) { var svcList, lbList []string services, err := utils.GetInformers().ServiceInformer.Lister().List(labels.Everything()) diff --git a/internal/nodes/avi_model_evh_nodes.go b/internal/nodes/avi_model_evh_nodes.go index cef0ba801..7a325fbdc 100644 --- a/internal/nodes/avi_model_evh_nodes.go +++ b/internal/nodes/avi_model_evh_nodes.go @@ -15,6 +15,7 @@ package nodes import ( + "bytes" "context" "encoding/json" "fmt" @@ -672,6 +673,15 @@ func (v *AviEvhVsNode) CalculateCheckSum() { } } + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict { + // Why do we need to change checksum of VS? As we are appending hostname--> list of ingresses mapping + // so we need to have updated list at vs metadata so that during AKO bootup we will have updated list + b := new(bytes.Buffer) + for key, val := range v.ServiceMetadata.HostToNamespaceIngressName { + fmt.Fprintf(b, "%s=%s", key, val) + } + checksumStringSlice = append(checksumStringSlice, fmt.Sprint(utils.Hash(b.String()))) + } // Note: Changing the order of strings being appended, while computing vsRefs and checksum, // will change the eventual checksum Hash. @@ -1046,8 +1056,13 @@ func (o *AviObjectGraph) BuildPolicyPGPoolsForEVH(vsNode []*AviEvhVsNode, childN func ProcessInsecureHostsForEVH(routeIgrObj RouteIngressModel, key string, parsedIng IngressConfig, modelList *[]string, Storedhosts map[string]*objects.RouteIngrhost, hostsMap map[string]*objects.RouteIngrhost) { utils.AviLog.Debugf("key: %s, msg: Storedhosts before processing insecurehosts: %s", key, utils.Stringify(Storedhosts)) + //flagIngToProcess := true for host, pathsvcmap := range parsedIng.IngressHostMap { // Remove this entry from storedHosts. First check if the host exists in the stored map or not. + flag := EnqueueIng(key, routeIgrObj.GetNamespace(), host, routeIgrObj.GetName()) + if !flag { + continue + } hostData, found := Storedhosts[host] if found && hostData.InsecurePolicy != lib.PolicyNone { // Verify the paths and take out the paths that are not need. @@ -1085,6 +1100,7 @@ func ProcessInsecureHostsForEVH(routeIgrObj RouteIngressModel, key string, parse modelGraph := aviModel.(*AviObjectGraph) modelGraph.BuildModelGraphForInsecureEVH(routeIgrObj, host, infraSetting, key, pathsvcmap) + //if flagIngToProcess { if len(vsNode) > 0 && found { // if vsNode already exists, check for updates via AviInfraSetting if infraSetting != nil { @@ -1095,9 +1111,11 @@ func ProcessInsecureHostsForEVH(routeIgrObj RouteIngressModel, key string, parse if !utils.HasElem(modelList, modelName) && changedModel { *modelList = append(*modelList, modelName) } + //} } utils.AviLog.Debugf("key: %s, msg: Storedhosts after processing insecurehosts: %s", key, utils.Stringify(Storedhosts)) + //return flagIngToProcess } func (o *AviObjectGraph) BuildModelGraphForInsecureEVH(routeIgrObj RouteIngressModel, host string, infraSetting *akov1beta1.AviInfraSetting, key string, pathsvcmap HostMetadata) { @@ -1117,6 +1135,9 @@ func (o *AviObjectGraph) BuildModelGraphForInsecureEVH(routeIgrObj RouteIngressM // Populate the hostmap with empty secret for insecure ingress PopulateIngHostMap(namespace, host, ingName, "", pathsvcmap) _, ingressHostMap := SharedHostNameLister().Get(host) + hostToIngressMap := make(map[string][]string) + // host --> list of namespace/ingress-name + hostToIngressMap[host] = ingressHostMap.GetIngressesForHostName() if lib.VIPPerNamespace() { SharedHostNameLister().SaveNamespace(host, routeIgrObj.GetNamespace()) @@ -1132,9 +1153,10 @@ func (o *AviObjectGraph) BuildModelGraphForInsecureEVH(routeIgrObj RouteIngressM EVHParent: false, EvhHostName: host, ServiceMetadata: lib.ServiceMetadataObj{ - NamespaceIngressName: ingressHostMap.GetIngressesForHostName(), - Namespace: namespace, - HostNames: hostSlice, + NamespaceIngressName: ingressHostMap.GetIngressesForHostName(), + Namespace: namespace, + HostNames: hostSlice, + HostToNamespaceIngressName: hostToIngressMap, }, } } else { @@ -1142,6 +1164,7 @@ func (o *AviObjectGraph) BuildModelGraphForInsecureEVH(routeIgrObj RouteIngressM evhNode.ServiceMetadata.NamespaceIngressName = ingressHostMap.GetIngressesForHostName() evhNode.ServiceMetadata.Namespace = namespace evhNode.ServiceMetadata.HostNames = hostSlice + evhNode.ServiceMetadata.HostToNamespaceIngressName = hostToIngressMap } evhNode.ServiceEngineGroup = lib.GetSEGName() evhNode.VrfContext = lib.GetVrf() @@ -1151,6 +1174,7 @@ func (o *AviObjectGraph) BuildModelGraphForInsecureEVH(routeIgrObj RouteIngressM vsNode[0].ServiceMetadata.NamespaceIngressName = ingressHostMap.GetIngressesForHostName() vsNode[0].ServiceMetadata.Namespace = namespace vsNode[0].ServiceMetadata.HostNames = hostSlice + vsNode[0].ServiceMetadata.HostToNamespaceIngressName = hostToIngressMap vsNode[0].AviMarkers = lib.PopulateVSNodeMarkers(namespace, host, infraSettingName) } @@ -1368,6 +1392,11 @@ func ProcessSecureHostsForEVH(routeIgrObj RouteIngressModel, key string, parsedI for _, tlssetting := range parsedIng.TlsCollection { locEvhHostMap := evhNodeHostName(routeIgrObj, tlssetting, routeIgrObj.GetName(), routeIgrObj.GetNamespace(), key, fullsync, sharedQueue, modelList) for host, newPathSvc := range locEvhHostMap { + // Remove this entry from storedHosts. First check if the host exists in the stored map or not. + flag := EnqueueIng(key, routeIgrObj.GetNamespace(), host, routeIgrObj.GetName()) + if !flag { + continue + } // Remove this entry from storedHosts. First check if the host exists in the stored map or not. hostData, found := Storedhosts[host] if found && hostData.InsecurePolicy == lib.PolicyAllow { @@ -1409,6 +1438,7 @@ func evhNodeHostName(routeIgrObj RouteIngressModel, tlssetting TlsSettings, ingN hostPathSvcMap[host] = paths.ingressHPSvc PopulateIngHostMap(namespace, host, ingName, tlssetting.SecretName, paths) + _, ingressHostMap := SharedHostNameLister().Get(host) if lib.VIPPerNamespace() { @@ -1463,7 +1493,8 @@ func (o *AviObjectGraph) BuildModelGraphForSecureEVH(routeIgrObj RouteIngressMod if lib.IsSecretAviCertRef(evhSecretName) { certsBuilt = true } - + hostToIngressMap := make(map[string][]string) + hostToIngressMap[host] = ingressHostMap.GetIngressesForHostName() var infraSettingName string if infraSetting != nil && !lib.IsInfraSettingNSScoped(infraSetting.Name, namespace) { infraSettingName = infraSetting.Name @@ -1479,9 +1510,10 @@ func (o *AviObjectGraph) BuildModelGraphForSecureEVH(routeIgrObj RouteIngressMod EVHParent: false, EvhHostName: host, ServiceMetadata: lib.ServiceMetadataObj{ - NamespaceIngressName: ingressHostMap.GetIngressesForHostName(), - Namespace: namespace, - HostNames: hosts, + NamespaceIngressName: ingressHostMap.GetIngressesForHostName(), + Namespace: namespace, + HostNames: hosts, + HostToNamespaceIngressName: hostToIngressMap, }, } } else { @@ -1489,6 +1521,7 @@ func (o *AviObjectGraph) BuildModelGraphForSecureEVH(routeIgrObj RouteIngressMod evhNode.ServiceMetadata.NamespaceIngressName = ingressHostMap.GetIngressesForHostName() evhNode.ServiceMetadata.Namespace = namespace evhNode.ServiceMetadata.HostNames = hosts + evhNode.ServiceMetadata.HostToNamespaceIngressName = hostToIngressMap } evhNode.ApplicationProfile = utils.DEFAULT_L7_APP_PROFILE evhNode.ServiceEngineGroup = lib.GetSEGName() @@ -1498,6 +1531,7 @@ func (o *AviObjectGraph) BuildModelGraphForSecureEVH(routeIgrObj RouteIngressMod vsNode[0].ServiceMetadata.NamespaceIngressName = ingressHostMap.GetIngressesForHostName() vsNode[0].ServiceMetadata.Namespace = namespace vsNode[0].ServiceMetadata.HostNames = hosts + vsNode[0].ServiceMetadata.HostToNamespaceIngressName = hostToIngressMap vsNode[0].AddSSLPort(key) vsNode[0].Secure = true vsNode[0].ApplicationProfile = utils.DEFAULT_L7_SECURE_APP_PROFILE @@ -1737,7 +1771,7 @@ func DeleteStaleDataForEvh(routeIgrObj RouteIngressModel, key string, modelList // Delete the pool corresponding to this host isPassthroughVS := false if hostData.SecurePolicy == lib.PolicyEdgeTerm { - aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, removeFqdn, removeRedir, removeRouteIngData, true) + aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, removeFqdn, removeRedir, removeRouteIngData, true, false) } else if hostData.SecurePolicy == lib.PolicyPass { isPassthroughVS = true aviModel.(*AviObjectGraph).DeleteObjectsForPassthroughHost(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, infraSettingName, key, true, true, true) @@ -1746,7 +1780,7 @@ func DeleteStaleDataForEvh(routeIgrObj RouteIngressModel, key string, modelList if isPassthroughVS { aviModel.(*AviObjectGraph).DeletePoolForHostname(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, true, true, false) } else { - aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, removeFqdn, removeRedir, removeRouteIngData, false) + aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, removeFqdn, removeRedir, removeRouteIngData, false, false) } } changedModel := saveAviModel(modelName, aviModel.(*AviObjectGraph), key) @@ -1913,7 +1947,7 @@ func (o *AviObjectGraph) RemovePGNodeRefsForEvh(pgName string, vsNode *AviEvhVsN utils.AviLog.Debugf("After removing the pg nodes are: %s", utils.Stringify(vsNode.PoolGroupRefs)) } -func (o *AviObjectGraph) manipulateEVHVsNode(vsNode *AviEvhVsNode, ingName, namespace, hostname string, pathSvc map[string][]string, infraSettingName, key string) { +func (o *AviObjectGraph) manipulateEVHVsNode(vsNode *AviEvhVsNode, ingName, namespace, hostname string, pathSvc map[string][]string, infraSettingName, key string, deleteHostMapEntry bool) { for path, services := range pathSvc { pgName := lib.GetEvhPGName(ingName, namespace, hostname, path, infraSettingName, vsNode.Dedicated) pgNode := vsNode.GetPGForVSByName(pgName) @@ -1929,14 +1963,32 @@ func (o *AviObjectGraph) manipulateEVHVsNode(vsNode *AviEvhVsNode, ingName, name httppolname := lib.GetSniHttpPolName(namespace, hostname, infraSettingName) hppmapname := lib.GetEvhPGName(ingName, namespace, hostname, path, infraSettingName, vsNode.Dedicated) o.RemoveHTTPRefsStringGroupsFromEvh(httppolname, hppmapname, vsNode) + if deleteHostMapEntry { + // This logic is to replace ingressname from servicemetadata hostToIngress map. + utils.AviLog.Debugf("HostIngmap before update is :%v", utils.Stringify(vsNode.ServiceMetadata.HostToNamespaceIngressName)) + hostToNamespaceIngMap := vsNode.ServiceMetadata.HostToNamespaceIngressName[hostname] + if len(hostToNamespaceIngMap) != 0 { + ingNameToReplace := fmt.Sprintf("%s/%s", namespace, ingName) + index := 0 + for i, ingNSName := range hostToNamespaceIngMap { + if ingNSName == ingNameToReplace { + index = i + break + } + } + hostToNamespaceIngMap = append(hostToNamespaceIngMap[:index], hostToNamespaceIngMap[index+1:]...) + vsNode.ServiceMetadata.HostToNamespaceIngressName[hostname] = hostToNamespaceIngMap + utils.AviLog.Debugf("HostIngmap after update is :%v", utils.Stringify(vsNode.ServiceMetadata.HostToNamespaceIngressName)) + } + } } } } } } -func (o *AviObjectGraph) ManipulateEvhNode(currentEvhNodeName, ingName, namespace, hostname string, pathSvc map[string][]string, vsNode []*AviEvhVsNode, infraSettingName, key string) bool { +func (o *AviObjectGraph) ManipulateEvhNode(currentEvhNodeName, ingName, namespace, hostname string, pathSvc map[string][]string, vsNode []*AviEvhVsNode, infraSettingName, key string, deleteHostMapEntry bool) bool { if vsNode[0].Dedicated { - o.manipulateEVHVsNode(vsNode[0], ingName, namespace, hostname, pathSvc, infraSettingName, key) + o.manipulateEVHVsNode(vsNode[0], ingName, namespace, hostname, pathSvc, infraSettingName, key, deleteHostMapEntry) if len(vsNode[0].PoolGroupRefs) == 0 { // Remove the evhhost mapping SharedHostNameLister().Delete(hostname) @@ -1948,7 +2000,7 @@ func (o *AviObjectGraph) ManipulateEvhNode(currentEvhNodeName, ingName, namespac if currentEvhNodeName != modelEvhNode.Name { continue } - o.manipulateEVHVsNode(modelEvhNode, ingName, namespace, hostname, pathSvc, infraSettingName, key) + o.manipulateEVHVsNode(modelEvhNode, ingName, namespace, hostname, pathSvc, infraSettingName, key, deleteHostMapEntry) // After going through the paths, if the EVH node does not have any PGs - then delete it. if len(modelEvhNode.PoolGroupRefs) == 0 { RemoveEvhInModel(currentEvhNodeName, vsNode, key) @@ -1976,7 +2028,7 @@ func (o *AviObjectGraph) GetAviPoolNodesByIngressForEvh(tenant string, ingName s return aviPool } -func (o *AviObjectGraph) DeletePoolForHostnameForEvh(vsName, hostname string, routeIgrObj RouteIngressModel, pathSvc map[string][]string, key, infraSettingName string, removeFqdn, removeRedir, removeRouteIngData, secure bool) bool { +func (o *AviObjectGraph) DeletePoolForHostnameForEvh(vsName, hostname string, routeIgrObj RouteIngressModel, pathSvc map[string][]string, key, infraSettingName string, removeFqdn, removeRedir, removeRouteIngData, secure, deleteHostMapEntry bool) bool { o.Lock.Lock() defer o.Lock.Unlock() @@ -1995,7 +2047,7 @@ func (o *AviObjectGraph) DeletePoolForHostnameForEvh(vsName, hostname string, ro evhNodeName := lib.GetEvhNodeName(hostname, infraSettingName) utils.AviLog.Infof("key: %s, msg: EVH node to delete: %s", key, evhNodeName) if removeRouteIngData { - keepEvh = o.ManipulateEvhNode(evhNodeName, ingName, namespace, hostname, pathSvc, vsNode, infraSettingName, key) + keepEvh = o.ManipulateEvhNode(evhNodeName, ingName, namespace, hostname, pathSvc, vsNode, infraSettingName, key, deleteHostMapEntry) } if !keepEvh { // Delete the cert ref for the host @@ -2114,12 +2166,12 @@ func RouteIngrDeletePoolsByHostnameForEvh(routeIgrObj RouteIngressModel, namespa // Delete the pool corresponding to this host if hostData.SecurePolicy == lib.PolicyEdgeTerm { - deleteVS = aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, true, true, true, true) + deleteVS = aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, true, true, true, true, true) } else if hostData.SecurePolicy == lib.PolicyPass { aviModel.(*AviObjectGraph).DeleteObjectsForPassthroughHost(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, infraSettingName, key, true, true, true) } if hostData.InsecurePolicy == lib.PolicyAllow { - deleteVS = aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, true, true, true, false) + deleteVS = aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, true, true, true, false, true) } if !deleteVS { ok := saveAviModel(modelName, aviModel.(*AviObjectGraph), key) @@ -2176,7 +2228,7 @@ func DeleteStaleDataForModelChangeForEvh(routeIgrObj RouteIngressModel, namespac // Delete the pool corresponding to this host isPassthroughVS := false if hostData.SecurePolicy == lib.PolicyEdgeTerm { - aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, true, true, true, true) + aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, true, true, true, true, false) } else if hostData.SecurePolicy == lib.PolicyPass { isPassthroughVS = true aviModel.(*AviObjectGraph).DeleteObjectsForPassthroughHost(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, infraSettingName, key, true, true, true) @@ -2185,7 +2237,7 @@ func DeleteStaleDataForModelChangeForEvh(routeIgrObj RouteIngressModel, namespac if isPassthroughVS { aviModel.(*AviObjectGraph).DeletePoolForHostname(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, true, true, false) } else { - aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, true, true, true, false) + aviModel.(*AviObjectGraph).DeletePoolForHostnameForEvh(shardVsName.Name, host, routeIgrObj, hostData.PathSvc, key, infraSettingName, true, true, true, false, false) } } diff --git a/internal/nodes/avi_model_l7_hostname_shard.go b/internal/nodes/avi_model_l7_hostname_shard.go index a44945495..a5f6a4472 100644 --- a/internal/nodes/avi_model_l7_hostname_shard.go +++ b/internal/nodes/avi_model_l7_hostname_shard.go @@ -53,6 +53,7 @@ func (o *AviObjectGraph) BuildDedicatedL7VSGraphHostNameShard(vsName, hostname s // Populate the hostmap with empty secret for insecure ingress PopulateIngHostMap(namespace, hostname, ingName, "", pathsvcMap) _, ingressHostMap := SharedHostNameLister().Get(hostname) + vsNode[0].ServiceMetadata.NamespaceIngressName = ingressHostMap.GetIngressesForHostName() vsNode[0].ServiceMetadata.Namespace = namespace vsNode[0].ServiceMetadata.HostNames = pathFQDNs @@ -313,6 +314,7 @@ func (o *AviObjectGraph) BuildL7VSGraphHostNameShard(vsName, hostname string, ro // Processsing insecure ingress if !utils.HasElem(vsNode[0].VSVIPRefs[0].FQDNs, hostname) { vsNode[0].VSVIPRefs[0].FQDNs = append(vsNode[0].VSVIPRefs[0].FQDNs, hostname) + // combine maps of each hostname. } // Check poolname length, if >255, don't add it. if lib.CheckObjectNameLength(poolName, lib.Pool) { @@ -604,6 +606,7 @@ func sniNodeHostName(routeIgrObj RouteIngressModel, tlssetting TlsSettings, ingN var sniHosts []string hostPathSvcMap[sniHost] = paths.ingressHPSvc PopulateIngHostMap(namespace, sniHost, ingName, tlssetting.SecretName, paths) + _, ingressHostMap := SharedHostNameLister().Get(sniHost) sniHosts = append(sniHosts, sniHost) _, shardVsName := DeriveShardVS(sniHost, key, routeIgrObj) diff --git a/internal/nodes/avi_model_nodes.go b/internal/nodes/avi_model_nodes.go index 66ee195b0..817363482 100644 --- a/internal/nodes/avi_model_nodes.go +++ b/internal/nodes/avi_model_nodes.go @@ -216,7 +216,6 @@ func (v *AviEvhVsNode) CalculateForGraphChecksum() uint32 { for _, stringGroup := range v.StringGroupRefs { checksumStringSlice = append(checksumStringSlice, fmt.Sprint(stringGroup.GetCheckSum())) } - return utils.Hash(strings.Join(checksumStringSlice, ":")) } diff --git a/internal/nodes/avi_model_routeingr_hostname_shard.go b/internal/nodes/avi_model_routeingr_hostname_shard.go index 1129c816f..9f20b95d0 100644 --- a/internal/nodes/avi_model_routeingr_hostname_shard.go +++ b/internal/nodes/avi_model_routeingr_hostname_shard.go @@ -121,6 +121,7 @@ func HostNameShardAndPublish(objType, objname, namespace, key string, fullsync b return } + // TODO: These functions will return true or false. Depeding upon that we should update hostcache to have proper sync // Process insecure routes first. ProcessInsecureHosts(routeIgrObj, key, parsedIng, &modelList, Storedhosts, hostsMap) diff --git a/internal/nodes/hostname_cache.go b/internal/nodes/hostname_cache.go index d54f27080..2033564fc 100644 --- a/internal/nodes/hostname_cache.go +++ b/internal/nodes/hostname_cache.go @@ -18,6 +18,7 @@ import ( "strings" "sync" + "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/lib" "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/objects" akov1beta1 "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/apis/ako/v1beta1" @@ -172,11 +173,11 @@ func (h *HostNamePathStore) DeleteHostPathStore(host string) { h.hostNamePathStore.Delete(host) } +// added multiple layer of validation. Useful in multihost ingress func PopulateIngHostMap(namespace, hostName, ingName, secretName string, pathsvcMap HostMetadata) { hostMap := HostNamePathSecrets{paths: getPaths(pathsvcMap.ingressHPSvc), secretName: secretName} found, ingressHostMap := SharedHostNameLister().Get(hostName) if found { - // Replace the ingress map for this host. ingressHostMap.HostNameMap[namespace+"/"+ingName] = hostMap } else { // Create the map @@ -185,3 +186,23 @@ func PopulateIngHostMap(namespace, hostName, ingName, secretName string, pathsvc } SharedHostNameLister().Save(hostName, ingressHostMap) } + +func EnqueueIng(key, namespace, hostName, ingName string) bool { + if lib.AKOControlConfig().GetAKOFQDNReusePolicy() != lib.FQDNReusePolicyStrict { + return true + } + found, ingressHostMap := SharedHostNameLister().Get(hostName) + if found { + ingresses := ingressHostMap.GetIngressesForHostName() + if len(ingresses) > 0 { + nsIngressname := strings.Split(ingresses[0], "/") + if len(nsIngressname) > 0 { + if nsIngressname[0] != namespace { + utils.AviLog.Debugf("key: %s, msg: Host %s already claimed. Not processing ingress.", key, hostName) + return false + } + } + } + } + return true +} diff --git a/internal/objects/uniquehostname.go b/internal/objects/uniquehostname.go new file mode 100644 index 000000000..2a87ccb56 --- /dev/null +++ b/internal/objects/uniquehostname.go @@ -0,0 +1,196 @@ +/* + * Copyright 2024 VMware, Inc. + * All Rights Reserved. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package objects + +import ( + "strings" + "sync" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/utils" +) + +var uniqueHostListerInstance *UniqueHostNamespaceLister +var uniqueHostListerOnce sync.Once + +func SharedUniqueNamespaceLister() *UniqueHostNamespaceLister { + uniqueHostListerOnce.Do(func() { + uniqueHostListerInstance = &UniqueHostNamespaceLister{ + HostnameToRoutesStore: NewObjectMapStore(), + RouteToHostnamesStore: NewObjectMapStore(), + } + }) + return uniqueHostListerInstance +} + +type UniqueHostNamespaceLister struct { + HostnameLock sync.RWMutex + // hostname --> Active Routes + HostnameToRoutesStore *ObjectMapStore + // Route (namespace/name) --> Hostnames (not being used) + RouteToHostnamesStore *ObjectMapStore +} + +type RouteNamspaceName struct { + // each entry will be objecttype/route-namespace/route-name + RouteNSRouteName string + // store creation time + CreationTime metav1.Time +} + +// key --> Routenamespacename -> string +// value --> creation time --> string +type RouteList struct { + activeRoutes map[string]metav1.Time + // Commenting it for now. In future if we decide to dynamically update the lists, unblock following + /* + inactiveRoutes map[string]metav1.Time + displacedRoutes map[string]metav1.Time + */ +} + +// =======hostname to route mapping=== +// == This function should be called in add or update (in update also for newhost name ) for each hostname of the route +// Keeping signature same for future use. +func (n *UniqueHostNamespaceLister) UpdateHostnameToRoute(hostName string, route RouteNamspaceName) (bool, []string, []string) { + n.HostnameLock.Lock() + defer n.HostnameLock.Unlock() + var localRouteList RouteList + + // fetch existing data first + utils.AviLog.Debugf("Hostname is: %s", hostName) + utils.AviLog.Debugf("Current route that needs to be processed is : %v", utils.Stringify(route)) + present, routeList := n.HostnameToRoutesStore.Get(hostName) + utils.AviLog.Debugf("Details fetched from HostnameToRoutesStore is: Present: %v and routeList: %v", present, utils.Stringify(routeList)) + if !present { + // host not present, add it to activeRoute list + // Allocate map + localRouteList.activeRoutes = make(map[string]metav1.Time) + + localRouteList.activeRoutes[route.RouteNSRouteName] = route.CreationTime + n.HostnameToRoutesStore.AddOrUpdate(hostName, localRouteList) + utils.AviLog.Debugf("Active routes after appending: %v", utils.Stringify(localRouteList.activeRoutes)) + return true, nil, nil + } + existingRoutes := routeList.(RouteList) + // Check: route is presnt in active list or not + if _, found := existingRoutes.activeRoutes[route.RouteNSRouteName]; found { + utils.AviLog.Debugf("key %s found in active routes.", route.RouteNSRouteName) + // ingest only that route + return true, nil, nil + } + + // Key: not found in active , now compare namespace present in activeRoutes + sameNamespace := false + currentRouteNamespace, _ := ExtractNamespaceName(route.RouteNSRouteName) + // TODO: Debug statements can be reduced + utils.AviLog.Debugf("Namespace of current route is: %s", currentRouteNamespace) + + // go through each active route and compare namespace with active routes. + // Active route should contains routes from same namespace + // Need to check in case of delete of hostname, are we deleting all these list and entry from internal datastore. + for nsName, creationTime := range existingRoutes.activeRoutes { + utils.AviLog.Debugf("NamespaceName from active route list is: %v and creation timestamp: %v", nsName, creationTime) + existingActiveRoutesNamespace, _ := ExtractNamespaceName(nsName) + utils.AviLog.Debugf("Extracted out namespace is: %s", existingActiveRoutesNamespace) + if currentRouteNamespace == existingActiveRoutesNamespace { + // add it to active list + sameNamespace = true + break + } + } + // With new logic, we need to check length of active route list. if it is empty then, we need to take another namespace + + if sameNamespace { + utils.AviLog.Debugf("Namespace is same. Attaching Route %v to active route list", route.RouteNSRouteName) + existingRoutes.activeRoutes[route.RouteNSRouteName] = route.CreationTime + n.HostnameToRoutesStore.AddOrUpdate(hostName, existingRoutes) + // DO not pass anything. Just return true so to process given route + return true, nil, nil + } + utils.AviLog.Debugf("Route %v is not accepted.", route.RouteNSRouteName) + return false, nil, nil +} + +func ExtractNamespaceName(nsName string) (string, string) { + if nsName != "" { + // changed from Split to SplitN + splitStrings := strings.SplitN(nsName, "/", 3) + if len(splitStrings) == 3 { + return splitStrings[1], splitStrings[2] + } + if len(splitStrings) == 2 { + return splitStrings[0], splitStrings[1] + } + } + return "", "" +} + +// This has to be called for each hostname of the route +func (n *UniqueHostNamespaceLister) DeleteHostnameToRoute(hostName string, route RouteNamspaceName) ([]string, string) { + n.HostnameLock.Lock() + defer n.HostnameLock.Unlock() + present, routeList := n.HostnameToRoutesStore.Get(hostName) + if !present { + // hostname entry is not present in the list + utils.AviLog.Warnf("Returning as host name %s is not present in store", hostName) + return nil, "" + } + existingRoutes := routeList.(RouteList) + // Check route is present in active list + if _, flag := existingRoutes.activeRoutes[route.RouteNSRouteName]; flag { + // present + utils.AviLog.Debugf("RouteNS to be deleted: %v and Active routes are: %v", route.RouteNSRouteName, existingRoutes.activeRoutes) + delete(existingRoutes.activeRoutes, route.RouteNSRouteName) + + utils.AviLog.Debugf("After deleting Active routes are %v", utils.Stringify(existingRoutes.activeRoutes)) + if len(existingRoutes.activeRoutes) != 0 { + n.HostnameToRoutesStore.AddOrUpdate(hostName, existingRoutes) + // TODO: It will good fetch data and print it for debugging. + //_, temproutes := n.HostnameToRoutesStore.Get(hostName) + return nil, route.RouteNSRouteName + } + utils.AviLog.Debugf("Active route list is empty for hostname %s. Deleting map entry", hostName) + n.HostnameToRoutesStore.Delete(hostName) + } + + return nil, "" +} + +//=== route to hostname mapping (Not being used. Done for future use)==== + +// Add/update list of hostnames associated with given route +func (n *UniqueHostNamespaceLister) UpdateRouteToHostname(routeNSName string, hostNames []string) { + n.HostnameLock.Lock() + defer n.HostnameLock.Unlock() + n.RouteToHostnamesStore.AddOrUpdate(routeNSName, hostNames) +} + +func (n *UniqueHostNamespaceLister) GetRouteToHostname(routeName string) (bool, []string) { + n.HostnameLock.RLock() + defer n.HostnameLock.RUnlock() + found, hostNameList := n.RouteToHostnamesStore.Get(routeName) + if found { + return true, hostNameList.([]string) + } + return false, []string{} +} + +func (n *UniqueHostNamespaceLister) DeleteRouteToHostname(routeName string) { + n.HostnameLock.Lock() + defer n.HostnameLock.Unlock() + n.RouteToHostnamesStore.Delete(routeName) +} diff --git a/tests/evhtests/l7_evh_graph_test.go b/tests/evhtests/l7_evh_graph_test.go index 3bac08865..16311e258 100644 --- a/tests/evhtests/l7_evh_graph_test.go +++ b/tests/evhtests/l7_evh_graph_test.go @@ -2086,3 +2086,90 @@ func TestPortsForInsecureAndSecureEVH(t *testing.T) { KubeClient.CoreV1().Secrets("default").Delete(context.TODO(), secretName, metav1.DeleteOptions{}) TearDownTestForIngress(t, svcName, modelName) } + +func TestMultiIngressSameHostDifferentNamespaceForEvh(t *testing.T) { + g := gomega.NewGomegaWithT(t) + lib.AKOControlConfig().SetAKOFQDNReusePolicy("strict") + modelName, _ := GetModelName("foo.com", "default") + SetUpTestForIngress(t, modelName) + + ingrFake1 := (integrationtest.FakeIngress{ + Name: "ingress-multi1", + Namespace: "default", + DnsNames: []string{"foo.com"}, + Paths: []string{"/foo"}, + ServiceName: "avisvc", + }).Ingress() + + _, err := KubeClient.NetworkingV1().Ingresses("default").Create(context.TODO(), ingrFake1, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error in adding Ingress: %v", err) + } + + // red namespace + ingrFake2 := (integrationtest.FakeIngress{ + Name: "ingress-multi2", + Namespace: "red", + DnsNames: []string{"foo.com"}, + Paths: []string{"/bar"}, + ServiceName: "avisvc", + }).Ingress() + + ing_red, err := KubeClient.NetworkingV1().Ingresses("red").Create(context.TODO(), ingrFake2, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error in adding Ingress: %v", err) + } + // status should be empty. + // TODO: Events can be checked for ingress. + g.Expect(ing_red.Status.LoadBalancer).To(gomega.BeNil()) + integrationtest.PollForCompletion(t, modelName, 5) + found, aviModel := objects.SharedAviGraphLister().Get(modelName) + if found { + nodes := aviModel.(*avinodes.AviObjectGraph).GetAviEvhVS() + g.Expect(len(nodes)).To(gomega.Equal(1)) + g.Expect(nodes[0].Name).To(gomega.ContainSubstring("Shared-L7")) + g.Expect(nodes[0].Tenant).To(gomega.Equal("admin")) + g.Expect(len(nodes[0].PoolRefs)).To(gomega.Equal(0)) + + g.Eventually(func() int { + _, aviModel := objects.SharedAviGraphLister().Get(modelName) + nodes := aviModel.(*avinodes.AviObjectGraph).GetAviEvhVS() + return len(nodes[0].EvhNodes) + }, 10*time.Second).Should(gomega.Equal(1)) + g.Expect(nodes[0].EvhNodes[0].Name).To(gomega.Equal(lib.Encode("cluster--foo.com", lib.EVHVS))) + g.Expect(nodes[0].EvhNodes[0].EvhHostName).To(gomega.Equal("foo.com")) + g.Expect(nodes[0].EvhNodes[0].HttpPolicyRefs).Should(gomega.HaveLen(1)) + g.Expect(nodes[0].EvhNodes[0].HttpPolicyRefs[0].HppMap).Should(gomega.HaveLen(1)) + g.Expect(len(nodes[0].EvhNodes[0].HttpPolicyRefs[0].HppMap[0].Path), gomega.Equal(1)) + g.Expect(func() []string { + p := []string{ + nodes[0].EvhNodes[0].HttpPolicyRefs[0].HppMap[0].Path[0], + } + sort.Strings(p) + return p + }, gomega.Equal([]string{"/foo"})) + g.Expect(len(nodes[0].EvhNodes[0].PoolGroupRefs)).To(gomega.Equal(1)) + } else { + t.Fatalf("Could not find model: %s", modelName) + } + err = KubeClient.NetworkingV1().Ingresses("default").Delete(context.TODO(), "ingress-multi1", metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Couldn't DELETE the Ingress %v", err) + } + VerifyEvhPoolDeletion(t, g, aviModel, 0) + VerifyEvhIngressDeletion(t, g, aviModel, 1) + integrationtest.DetectModelChecksumChange(t, modelName, 5) + nodes := aviModel.(*avinodes.AviObjectGraph).GetAviEvhVS() + g.Expect(len(nodes)).To(gomega.Equal(1)) + g.Expect(len(nodes[0].EvhNodes)).To(gomega.Equal(0)) + + err = KubeClient.NetworkingV1().Ingresses("red").Delete(context.TODO(), "ingress-multi2", metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Couldn't DELETE the Ingress %v", err) + } + lib.AKOControlConfig().SetAKOFQDNReusePolicy("internamespaceallowed") + VerifyEvhPoolDeletion(t, g, aviModel, 0) + VerifyEvhIngressDeletion(t, g, aviModel, 0) + VerifyEvhVsCacheChildDeletion(t, g, cache.NamespaceName{Namespace: "admin", Name: modelName}) + TearDownTestForIngress(t, modelName) +}