From 81b40c30ef51f14871b44c453c26e5437b60c3c5 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Tue, 7 May 2024 13:37:03 +0800 Subject: [PATCH] support ovn eip nats dualstack (#3822) * support ovn eip nats dualstack --------- Signed-off-by: bobz965 --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 28 +++ dist/images/install.sh | 28 +++ pkg/apis/kubeovn/v1/types.go | 9 + pkg/controller/ovn_dnat.go | 235 ++++++++++++-------- pkg/controller/ovn_eip.go | 31 ++- pkg/controller/ovn_fip.go | 205 +++++++++-------- pkg/controller/ovn_snat.go | 210 ++++++++--------- pkg/util/const.go | 1 + pkg/webhook/ovn_nat_gateway.go | 26 +-- yamls/crd.yaml | 28 +++ 10 files changed, 479 insertions(+), 322 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index a305a378a51..d3e91e68db8 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -1268,9 +1268,15 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string + - jsonPath: .status.v6Eip + name: V6Eip + type: string - jsonPath: .status.v4Ip name: V4Ip type: string + - jsonPath: .status.v6Ip + name: V6Ip + type: string - jsonPath: .status.ready name: Ready type: boolean @@ -1325,6 +1331,8 @@ spec: type: string v4Ip: type: string + v6Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1353,9 +1361,15 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string + - jsonPath: .status.v6Eip + name: V6Eip + type: string - jsonPath: .status.v4IpCidr name: V4IpCidr type: string + - jsonPath: .status.v6IpCidr + name: V6IpCidr + type: string - jsonPath: .status.ready name: Ready type: boolean @@ -1370,8 +1384,12 @@ spec: type: boolean v4Eip: type: string + v6Eip: + type: string v4IpCidr: type: string + v6IpCidr: + type: string vpc: type: string conditions: @@ -1404,6 +1422,8 @@ spec: type: string v4IpCidr: type: string + v6IpCidr: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1438,9 +1458,15 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string + - jsonPath: .status.v6Eip + name: V6Eip + type: string - jsonPath: .status.v4Ip name: V4Ip type: string + - jsonPath: .status.v6Ip + name: V6Ip + type: string - jsonPath: .status.internalPort name: InternalPort type: string @@ -1512,6 +1538,8 @@ spec: type: string v4Ip: type: string + v6Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/dist/images/install.sh b/dist/images/install.sh index 40ccf769b0c..40c952a57fc 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -1496,9 +1496,15 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string + - jsonPath: .status.v6Eip + name: V6Eip + type: string - jsonPath: .status.v4Ip name: V4Ip type: string + - jsonPath: .status.v6Ip + name: V6Ip + type: string - jsonPath: .status.ready name: Ready type: boolean @@ -1553,6 +1559,8 @@ spec: type: string v4Ip: type: string + v6Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1581,9 +1589,15 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string + - jsonPath: .status.v6Eip + name: V6Eip + type: string - jsonPath: .status.v4IpCidr name: V4IpCidr type: string + - jsonPath: .status.v6IpCidr + name: V6IpCidr + type: string - jsonPath: .status.ready name: Ready type: boolean @@ -1598,8 +1612,12 @@ spec: type: boolean v4Eip: type: string + v6Eip: + type: string v4IpCidr: type: string + v6IpCidr: + type: string vpc: type: string conditions: @@ -1632,6 +1650,8 @@ spec: type: string v4IpCidr: type: string + v6IpCidr: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1666,9 +1686,15 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string + - jsonPath: .status.v6Eip + name: V6Eip + type: string - jsonPath: .status.v4Ip name: V4Ip type: string + - jsonPath: .status.v6Ip + name: V6Ip + type: string - jsonPath: .status.internalPort name: InternalPort type: string @@ -1740,6 +1766,8 @@ spec: type: string v4Ip: type: string + v6Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/pkg/apis/kubeovn/v1/types.go b/pkg/apis/kubeovn/v1/types.go index 27bb7d47084..c71fecb5251 100644 --- a/pkg/apis/kubeovn/v1/types.go +++ b/pkg/apis/kubeovn/v1/types.go @@ -1008,6 +1008,7 @@ type OvnFipSpec struct { IPName string `json:"ipName"` // vip, ip crd name Vpc string `json:"vpc"` V4Ip string `json:"v4Ip"` + V6Ip string `json:"v6Ip"` } // OvnFipCondition describes the state of an object at a certain point. @@ -1019,7 +1020,9 @@ type OvnFipStatus struct { // +patchStrategy=merge Vpc string `json:"vpc" patchStrategy:"merge"` V4Eip string `json:"v4Eip" patchStrategy:"merge"` + V6Eip string `json:"v6Eip" patchStrategy:"merge"` V4Ip string `json:"v4Ip" patchStrategy:"merge"` + V6Ip string `json:"v6Ip" patchStrategy:"merge"` Ready bool `json:"ready" patchStrategy:"merge"` // Conditions represents the latest state of the object @@ -1057,6 +1060,7 @@ type OvnSnatRuleSpec struct { IPName string `json:"ipName"` Vpc string `json:"vpc"` V4IpCidr string `json:"v4IpCidr"` // subnet cidr or pod ip address + V6IpCidr string `json:"v6IpCidr"` // subnet cidr or pod ip address } // OvnSnatRuleCondition describes the state of an object at a certain point. @@ -1068,7 +1072,9 @@ type OvnSnatRuleStatus struct { // +patchStrategy=merge Vpc string `json:"vpc" patchStrategy:"merge"` V4Eip string `json:"v4Eip" patchStrategy:"merge"` + V6Eip string `json:"v6Eip" patchStrategy:"merge"` V4IpCidr string `json:"v4IpCidr" patchStrategy:"merge"` + V6IpCidr string `json:"v6IpCidr" patchStrategy:"merge"` Ready bool `json:"ready" patchStrategy:"merge"` // Conditions represents the latest state of the object @@ -1109,6 +1115,7 @@ type OvnDnatRuleSpec struct { Protocol string `json:"protocol,omitempty"` Vpc string `json:"vpc"` V4Ip string `json:"v4Ip"` + V6Ip string `json:"v6Ip"` } // OvnDnatRuleCondition describes the state of an object at a certain point. @@ -1121,8 +1128,10 @@ type OvnDnatRuleStatus struct { // +patchStrategy=merge Vpc string `json:"vpc" patchStrategy:"merge"` V4Eip string `json:"v4Eip" patchStrategy:"merge"` + V6Eip string `json:"v6Eip" patchStrategy:"merge"` ExternalPort string `json:"externalPort"` V4Ip string `json:"v4Ip" patchStrategy:"merge"` + V6Ip string `json:"v6Ip" patchStrategy:"merge"` InternalPort string `json:"internalPort"` Protocol string `json:"protocol,omitempty"` IPName string `json:"ipName"` diff --git a/pkg/controller/ovn_dnat.go b/pkg/controller/ovn_dnat.go index d8f130d9732..8ca4c29a641 100644 --- a/pkg/controller/ovn_dnat.go +++ b/pkg/controller/ovn_dnat.go @@ -242,25 +242,20 @@ func (c *Controller) handleAddOvnDnatRule(key string) error { klog.Error(err) return err } - if cachedEip.Status.V4Ip == "" { - err := fmt.Errorf("failed to create v4 dnat %s, eip %s has no v4 ip", cachedDnat.Name, eipName) + + var v4Eip, v6Eip, internalV4Ip, internalV6Ip, subnetName, vpcName, ipName string + v4Eip = cachedEip.Status.V4Ip + v6Eip = cachedEip.Status.V6Ip + if v4Eip == "" && v6Eip == "" { + err := fmt.Errorf("failed to dnat %s, eip %s has no ip", cachedDnat.Name, eipName) klog.Error(err) return err } - if err := c.isOvnDnatDuplicated(eipName, key, cachedDnat.Spec.ExternalPort); err != nil { - klog.Errorf("failed to create dnat %s, %v", cachedDnat.Name, err) - return err - } - - // get dnat external ip, internal ip, vpc name - var internalV4Ip, subnetName, vpcName string - if cachedDnat.Spec.Vpc != "" { - vpcName = cachedDnat.Spec.Vpc - } - if cachedDnat.Spec.V4Ip != "" { - internalV4Ip = cachedDnat.Spec.V4Ip - } - if internalV4Ip == "" && cachedDnat.Spec.IPName != "" { + vpcName = cachedDnat.Spec.Vpc + internalV4Ip = cachedDnat.Spec.V4Ip + internalV6Ip = cachedDnat.Spec.V6Ip + ipName = cachedDnat.Spec.IPName + if ipName != "" { if cachedDnat.Spec.IPType == util.Vip { internalVip, err := c.virtualIpsLister.Get(cachedDnat.Spec.IPName) if err != nil { @@ -268,6 +263,7 @@ func (c *Controller) handleAddOvnDnatRule(key string) error { return err } internalV4Ip = internalVip.Status.V4ip + internalV6Ip = internalVip.Status.V6ip subnetName = internalVip.Spec.Subnet } else { internalIP, err := c.ipsLister.Get(cachedDnat.Spec.IPName) @@ -276,6 +272,7 @@ func (c *Controller) handleAddOvnDnatRule(key string) error { return err } internalV4Ip = internalIP.Spec.V4IPAddress + internalV6Ip = internalIP.Spec.V6IPAddress subnetName = internalIP.Spec.Subnet } subnet, err := c.subnetsLister.Get(subnetName) @@ -285,45 +282,75 @@ func (c *Controller) handleAddOvnDnatRule(key string) error { } vpcName = subnet.Spec.Vpc } - if internalV4Ip == "" { - err := fmt.Errorf("failed to create v4 dnat %s, no internal v4 ip", cachedDnat.Name) + if internalV4Ip == "" && internalV6Ip == "" { + err := fmt.Errorf("failed to create dnat %s, no internal ip", cachedDnat.Name) + klog.Error(err) + return err + } + if v4Eip == "" && v6Eip == "" { + err := fmt.Errorf("failed to create dnat %s, no eip", cachedDnat.Name) klog.Error(err) return err } if vpcName == "" { - err := fmt.Errorf("failed to create v4 dnat %s, no vpc", cachedDnat.Name) + err := fmt.Errorf("failed to create dnat %s, no vpc", cachedDnat.Name) klog.Error(err) return err } - if err = c.AddDnatRule(vpcName, cachedDnat.Name, cachedEip.Status.V4Ip, internalV4Ip, - cachedDnat.Spec.ExternalPort, cachedDnat.Spec.InternalPort, cachedDnat.Spec.Protocol); err != nil { - klog.Errorf("failed to create v4 dnat, %v", err) + var externalPort, internalPort, protocol string + externalPort = cachedDnat.Spec.ExternalPort + internalPort = cachedDnat.Spec.InternalPort + protocol = cachedDnat.Spec.Protocol + if externalPort == "" { + err := fmt.Errorf("failed to create dnat %s, no external port", cachedDnat.Name) + klog.Error(err) + return err + } + if err := c.isOvnDnatDuplicated(eipName, key, cachedDnat.Spec.ExternalPort); err != nil { + klog.Errorf("failed to create dnat %s, %v", cachedDnat.Name, err) + return err + } + if internalPort == "" { + err := fmt.Errorf("failed to create dnat %s, no internal port", cachedDnat.Name) + klog.Error(err) + return err + } + if protocol == "" { + err := fmt.Errorf("failed to create dnat %s, no protocol", cachedDnat.Name) + klog.Error(err) return err } + if internalV4Ip != "" && v4Eip != "" { + if err = c.AddDnatRule(vpcName, cachedDnat.Name, v4Eip, internalV4Ip, externalPort, internalPort, protocol); err != nil { + klog.Errorf("failed to create v4 dnat, %v", err) + return err + } + } + if internalV6Ip != "" && v6Eip != "" { + if err = c.AddDnatRule(vpcName, cachedDnat.Name, v6Eip, internalV6Ip, externalPort, internalPort, protocol); err != nil { + klog.Errorf("failed to create v6 dnat, %v", err) + return err + } + } if err := c.handleAddOvnDnatFinalizer(cachedDnat); err != nil { klog.Errorf("failed to add finalizer for ovn dnat %s, %v", cachedDnat.Name, err) return err } - // patch dnat eip relationship if err = c.natLabelAndAnnoOvnEip(eipName, cachedDnat.Name, vpcName); err != nil { klog.Errorf("failed to label dnat '%s' in eip %s, %v", cachedDnat.Name, eipName, err) return err } - if err = c.patchOvnDnatAnnotations(key, eipName); err != nil { klog.Errorf("failed to update annotations for dnat %s, %v", key, err) return err } - - if err = c.patchOvnDnatStatus(key, vpcName, cachedEip.Status.V4Ip, - internalV4Ip, true); err != nil { + if err = c.patchOvnDnatStatus(key, vpcName, v4Eip, v6Eip, internalV4Ip, internalV6Ip, true); err != nil { klog.Errorf("failed to patch status for dnat %s, %v", key, err) return err } - if err = c.patchOvnEipStatus(eipName, true); err != nil { klog.Errorf("failed to patch status for eip %s, %v", key, err) return err @@ -342,12 +369,25 @@ func (c *Controller) handleDelOvnDnatRule(key string) error { klog.Error(err) return err } - if cachedDnat.Status.Vpc != "" && cachedDnat.Status.V4Eip != "" && cachedDnat.Status.ExternalPort != "" { - if err = c.DelDnatRule(cachedDnat.Status.Vpc, cachedDnat.Name, - cachedDnat.Status.V4Eip, cachedDnat.Status.ExternalPort); err != nil { - klog.Errorf("failed to delete dnat %s, %v", key, err) - return err + if cachedDnat.Status.Vpc != "" { + if cachedDnat.Status.V4Eip != "" && cachedDnat.Status.ExternalPort != "" { + if err = c.DelDnatRule(cachedDnat.Status.Vpc, cachedDnat.Name, + cachedDnat.Status.V4Eip, cachedDnat.Status.ExternalPort); err != nil { + klog.Errorf("failed to delete v4 dnat %s, %v", key, err) + return err + } + } + if cachedDnat.Status.V6Eip != "" && cachedDnat.Status.ExternalPort != "" { + if err = c.DelDnatRule(cachedDnat.Status.Vpc, cachedDnat.Name, + cachedDnat.Status.V6Eip, cachedDnat.Status.ExternalPort); err != nil { + klog.Errorf("failed to delete v6 dnat %s, %v", key, err) + return err + } } + } else { + err := fmt.Errorf("failed to delete dnat %s, no vpc", cachedDnat.Name) + klog.Error(err) + return err } if err = c.handleDelOvnDnatFinalizer(cachedDnat); err != nil { klog.Errorf("failed to remove finalizer for ovn dnat %s, %v", cachedDnat.Name, err) @@ -368,7 +408,11 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { klog.Error(err) return err } - + if !cachedDnat.Status.Ready { + // create dnat only in add process, just check to error out here + klog.Infof("wait ovn dnat %s to be ready only in the handle add process", cachedDnat.Name) + return nil + } klog.Infof("handle update dnat %s", key) // check eip eipName := cachedDnat.Spec.OvnEip @@ -388,25 +432,19 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { klog.Error(err) return err } - if cachedEip.Status.V4Ip == "" { - err := fmt.Errorf("failed to create v4 dnat %s, eip %s has no v4 ip", cachedDnat.Name, eipName) + var v4Eip, v6Eip, internalV4Ip, internalV6Ip, subnetName, vpcName, ipName string + v4Eip = cachedEip.Status.V4Ip + v6Eip = cachedEip.Status.V6Ip + if v4Eip == "" && v6Eip == "" { + err := fmt.Errorf("failed to update dnat %s, eip %s has no ip", cachedDnat.Name, eipName) klog.Error(err) return err } - if err := c.isOvnDnatDuplicated(eipName, key, cachedDnat.Spec.ExternalPort); err != nil { - klog.Errorf("failed to create dnat %s, %v", cachedDnat.Name, err) - return err - } - - // get dnat external ip, internal ip, vpc name - var internalV4Ip, subnetName, vpcName string - if cachedDnat.Spec.Vpc != "" { - vpcName = cachedDnat.Spec.Vpc - } - if cachedDnat.Spec.V4Ip != "" { - internalV4Ip = cachedDnat.Spec.V4Ip - } - if internalV4Ip == "" && cachedDnat.Spec.IPName != "" { + vpcName = cachedDnat.Spec.Vpc + internalV4Ip = cachedDnat.Spec.V4Ip + internalV6Ip = cachedDnat.Spec.V6Ip + ipName = cachedDnat.Spec.IPName + if ipName != "" { if cachedDnat.Spec.IPType == util.Vip { internalVip, err := c.virtualIpsLister.Get(cachedDnat.Spec.IPName) if err != nil { @@ -414,6 +452,7 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { return err } internalV4Ip = internalVip.Status.V4ip + internalV6Ip = internalVip.Status.V6ip subnetName = internalVip.Spec.Subnet } else { internalIP, err := c.ipsLister.Get(cachedDnat.Spec.IPName) @@ -422,6 +461,7 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { return err } internalV4Ip = internalIP.Spec.V4IPAddress + internalV6Ip = internalIP.Spec.V6IPAddress subnetName = internalIP.Spec.Subnet } subnet, err := c.subnetsLister.Get(subnetName) @@ -431,46 +471,50 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { } vpcName = subnet.Spec.Vpc } - if internalV4Ip == "" { - err := fmt.Errorf("failed to create v4 dnat %s, no internal v4 ip", cachedDnat.Name) + // not support chang + if cachedDnat.Spec.ExternalPort != cachedDnat.Status.ExternalPort { + err := fmt.Errorf("not support change external port for dnat %s", cachedDnat.Name) klog.Error(err) return err } - if vpcName == "" { - err := fmt.Errorf("failed to create v4 dnat %s, no vpc", cachedDnat.Name) + if cachedDnat.Spec.InternalPort != cachedDnat.Status.InternalPort { + err := fmt.Errorf("not support change internal port for dnat %s", cachedDnat.Name) klog.Error(err) return err } - - dnat := cachedDnat.DeepCopy() - if dnat.Status.Ready { - klog.Infof("dnat change ip, old ip '%s', new ip %s", dnat.Status.V4Ip, cachedEip.Status.V4Ip) - if err = c.DelDnatRule(dnat.Status.Vpc, dnat.Name, dnat.Status.V4Eip, dnat.Status.ExternalPort); err != nil { - klog.Errorf("failed to delete dnat, %v", err) - return err - } - - if err = c.AddDnatRule(vpcName, dnat.Name, cachedEip.Status.V4Ip, internalV4Ip, - dnat.Spec.ExternalPort, dnat.Spec.InternalPort, dnat.Spec.Protocol); err != nil { - klog.Errorf("failed to create dnat, %v", err) - return err - } - - if err = c.natLabelAndAnnoOvnEip(eipName, dnat.Name, vpcName); err != nil { - klog.Errorf("failed to label dnat '%s' in eip %s, %v", dnat.Name, eipName, err) - return err - } - - if err = c.patchOvnDnatAnnotations(key, eipName); err != nil { - klog.Errorf("failed to update annotations for dnat %s, %v", key, err) - return err - } - - if err = c.patchOvnDnatStatus(key, vpcName, cachedEip.Status.V4Ip, internalV4Ip, true); err != nil { - klog.Errorf("failed to patch status for dnat '%s', %v", key, err) - return err - } - return nil + if cachedDnat.Spec.Protocol != cachedDnat.Status.Protocol { + err := fmt.Errorf("not support change protocol for dnat %s", cachedDnat.Name) + klog.Error(err) + return err + } + if v4Eip != cachedDnat.Status.V4Eip || v6Eip != cachedDnat.Status.V6Eip { + err := fmt.Errorf("not support change eip for dnat %s", cachedDnat.Name) + klog.Error(err) + return err + } + if internalV4Ip != cachedDnat.Status.V4Ip || internalV6Ip != cachedDnat.Status.V6Ip { + err := fmt.Errorf("not support change internal ip for dnat %s", cachedDnat.Name) + klog.Error(err) + return err + } + if vpcName != cachedDnat.Status.Vpc { + err := fmt.Errorf("not support change vpc for dnat %s", cachedDnat.Name) + klog.Error(err) + return err + } + if cachedDnat.Spec.InternalPort != cachedDnat.Status.InternalPort { + err := fmt.Errorf("not support change internal port for dnat %s", cachedDnat.Name) + klog.Error(err) + return err + } + if cachedDnat.Spec.ExternalPort != cachedDnat.Status.ExternalPort { + err := fmt.Errorf("not support change external port for dnat %s", cachedDnat.Name) + klog.Error(err) + return err + } + if err := c.isOvnDnatDuplicated(eipName, key, cachedDnat.Spec.ExternalPort); err != nil { + klog.Errorf("failed to update dnat %s, %v", cachedDnat.Name, err) + return err } return nil } @@ -520,7 +564,7 @@ func (c *Controller) patchOvnDnatAnnotations(key, eipName string) error { return nil } -func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, podIP string, ready bool) error { +func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, v6Eip, internalV4Ip, internalV6Ip string, ready bool) error { var ( oriDnat, dnat *kubeovnv1.OvnDnatRule err error @@ -544,6 +588,7 @@ func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, podIP string, ready needUpdateLabel = true dnat.Labels = map[string]string{ util.EipV4IpLabel: v4Eip, + util.EipV6IpLabel: v6Eip, } } else if dnat.Labels[util.EipV4IpLabel] != v4Eip { op = "replace" @@ -565,37 +610,38 @@ func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, podIP string, ready dnat.Status.Ready = ready changed = true } - if vpcName != "" && dnat.Status.Vpc != vpcName { dnat.Status.Vpc = vpcName changed = true } - if v4Eip != "" && dnat.Status.V4Eip != v4Eip { dnat.Status.V4Eip = v4Eip changed = true } - - if podIP != "" && dnat.Status.V4Ip != podIP { - dnat.Status.V4Ip = podIP + if v6Eip != "" && dnat.Status.V6Eip != v6Eip { + dnat.Status.V4Eip = v6Eip + changed = true + } + if internalV4Ip != "" && dnat.Status.V4Ip != internalV4Ip { + dnat.Status.V4Ip = internalV4Ip + changed = true + } + if internalV6Ip != "" && dnat.Status.V6Ip != internalV6Ip { + dnat.Status.V6Ip = internalV6Ip changed = true } - if ready && dnat.Spec.Protocol != "" && dnat.Status.Protocol != dnat.Spec.Protocol { dnat.Status.Protocol = dnat.Spec.Protocol changed = true } - if ready && dnat.Spec.IPName != "" && dnat.Spec.IPName != dnat.Status.IPName { dnat.Status.IPName = dnat.Spec.IPName changed = true } - if ready && dnat.Spec.InternalPort != "" && dnat.Status.InternalPort != dnat.Spec.InternalPort { dnat.Status.InternalPort = dnat.Spec.InternalPort changed = true } - if ready && dnat.Spec.ExternalPort != "" && dnat.Status.ExternalPort != dnat.Spec.ExternalPort { dnat.Status.ExternalPort = dnat.Spec.ExternalPort changed = true @@ -647,7 +693,6 @@ func (c *Controller) DelDnatRule(vpcName, dnatName, externalIP, externalPort str err error ) externalEndpoint = net.JoinHostPort(externalIP, externalPort) - if err = c.OVNNbClient.LoadBalancerDeleteVip(dnatName, externalEndpoint, ignoreHealthCheck); err != nil { klog.Errorf("delete loadBalancer vips %s: %v", externalEndpoint, err) return err diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index 32eb574bc5c..990eeaeefdf 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -288,13 +288,32 @@ func (c *Controller) handleUpdateOvnEip(key string) error { klog.Error(err) return err } + if !cachedEip.Status.Ready { + // create eip only in add process, just check to error out here + klog.Infof("wait ovn eip %s to be ready only in the handle add process", cachedEip.Name) + return nil + } klog.Infof("handle update ovn eip %s", cachedEip.Name) - if cachedEip.Spec.Type != util.OvnEipTypeLSP { - // node ext gw use lsp eip, has a nic on gw node, so left node to make it ready - if err = c.patchOvnEipStatus(key, true); err != nil { - klog.Errorf("failed to patch ovn eip %s: %v", key, err) - return err - } + // not support change + if cachedEip.Status.V4Ip != cachedEip.Spec.V4Ip { + err := fmt.Errorf("not support change v4 ip for ovn eip %s", cachedEip.Name) + klog.Error(err) + return err + } + if cachedEip.Status.V6Ip != cachedEip.Spec.V6Ip { + err := fmt.Errorf("not support change v6 ip for ovn eip %s", cachedEip.Name) + klog.Error(err) + return err + } + if cachedEip.Status.MacAddress != cachedEip.Spec.MacAddress { + err := fmt.Errorf("not support change mac address for ovn eip %s", cachedEip.Name) + klog.Error(err) + return err + } + if cachedEip.Status.Type != cachedEip.Spec.Type { + err := fmt.Errorf("not support change type for ovn eip %s", cachedEip.Name) + klog.Error(err) + return err } return nil } diff --git a/pkg/controller/ovn_fip.go b/pkg/controller/ovn_fip.go index 164af1bae2b..1ef9e40ce6a 100644 --- a/pkg/controller/ovn_fip.go +++ b/pkg/controller/ovn_fip.go @@ -224,43 +224,41 @@ func (c *Controller) handleAddOvnFip(key string) error { klog.Error(err) return err } - if cachedEip.Status.V4Ip == "" { - err := fmt.Errorf("failed to create v4 fip %s, eip %s has no v4 ip", cachedFip.Name, eipName) - klog.Error(err) - return err - } + var v4Eip, v6Eip, v4IP, v6IP string + v4Eip = cachedEip.Status.V4Ip + v6Eip = cachedEip.Status.V6Ip if err = c.isOvnFipDuplicated(key, cachedEip.Spec.V4Ip); err != nil { err = fmt.Errorf("failed to add fip %s, %v", key, err) klog.Error(err) return err } - var mac, internalV4Ip, subnetName, vpcName string - if cachedFip.Spec.Vpc != "" { - vpcName = cachedFip.Spec.Vpc - } - if cachedFip.Spec.V4Ip != "" { - internalV4Ip = cachedFip.Spec.V4Ip - } - if internalV4Ip == "" && cachedFip.Spec.IPName != "" { + var mac, subnetName, vpcName, ipName string + vpcName = cachedFip.Spec.Vpc + v4IP = cachedFip.Spec.V4Ip + v6IP = cachedFip.Spec.V6Ip + ipName = cachedFip.Spec.IPName + if ipName != "" { if cachedFip.Spec.IPType == util.Vip { - internalVip, err := c.virtualIpsLister.Get(cachedFip.Spec.IPName) + internalVip, err := c.virtualIpsLister.Get(ipName) if err != nil { - klog.Errorf("failed to get vip %s, %v", cachedFip.Spec.IPName, err) + klog.Errorf("failed to get vip %s, %v", ipName, err) return err } - internalV4Ip = internalVip.Status.V4ip + v4IP = internalVip.Status.V4ip + v6IP = internalVip.Status.V6ip subnetName = internalVip.Spec.Subnet // though vip lsp has its mac, vip always use its parent lsp nic mac // and vip could float to different parent lsp nic // all vip its parent lsp acl should allow the vip ip } else { - internalIP, err := c.ipsLister.Get(cachedFip.Spec.IPName) + internalIP, err := c.ipsLister.Get(ipName) if err != nil { - klog.Errorf("failed to get ip %s, %v", cachedFip.Spec.IPName, err) + klog.Errorf("failed to get ip %s, %v", ipName, err) return err } - internalV4Ip = internalIP.Spec.V4IPAddress + v4IP = internalIP.Spec.V4IPAddress + v6IP = internalIP.Spec.V6IPAddress subnetName = internalIP.Spec.Subnet mac = internalIP.Spec.MacAddress // mac is necessary while using distributed router fip, fip use lsp its mac @@ -278,22 +276,34 @@ func (c *Controller) handleAddOvnFip(key string) error { return err } } - if internalV4Ip == "" { - err := fmt.Errorf("failed to create v4 fip %s, no internal v4 ip", cachedFip.Name) + if vpcName == "" { + err := fmt.Errorf("failed to create fip %s, no vpc", cachedFip.Name) klog.Error(err) return err } - if vpcName == "" { - err := fmt.Errorf("failed to create v4 fip %s, no vpc", cachedFip.Name) + if v4IP == "" && v6IP == "" { + err := fmt.Errorf("failed to create fip %s, no internal ip", cachedFip.Name) + klog.Error(err) + return err + } + if v4Eip == "" && v6Eip == "" { + err := fmt.Errorf("failed to create fip %s, no external ip", cachedFip.Name) klog.Error(err) return err } // ovn add fip options := map[string]string{"staleless": strconv.FormatBool(c.ExternalGatewayType == kubeovnv1.GWDistributedType)} - if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeDNATAndSNAT, cachedEip.Status.V4Ip, - internalV4Ip, mac, cachedFip.Spec.IPName, options); err != nil { - klog.Errorf("failed to create v4 fip, %v", err) - return err + if v4IP != "" && v4Eip != "" { + if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeDNATAndSNAT, v4Eip, v4IP, mac, cachedFip.Spec.IPName, options); err != nil { + klog.Errorf("failed to create v4 fip, %v", err) + return err + } + } + if v6Eip == "" && v6IP != "" { + if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeDNATAndSNAT, v6Eip, v6IP, mac, cachedFip.Spec.IPName, options); err != nil { + klog.Errorf("failed to create v6 fip, %v", err) + return err + } } if err = c.handleAddOvnFipFinalizer(cachedFip); err != nil { @@ -310,8 +320,7 @@ func (c *Controller) handleAddOvnFip(key string) error { klog.Errorf("failed to update label for fip %s, %v", key, err) return err } - if err = c.patchOvnFipStatus(key, vpcName, cachedEip.Status.V4Ip, - internalV4Ip, true); err != nil { + if err = c.patchOvnFipStatus(key, vpcName, v4Eip, v4IP, true); err != nil { klog.Errorf("failed to patch status for fip %s, %v", key, err) return err } @@ -331,6 +340,11 @@ func (c *Controller) handleUpdateOvnFip(key string) error { klog.Error(err) return err } + if !cachedFip.Status.Ready { + // create fip only in add process, just check to error out here + klog.Infof("wait ovn fip %s to be ready only in the handle add process", cachedFip.Name) + return nil + } klog.Infof("handle update fip %s", key) // check eip eipName := cachedFip.Spec.OvnEip @@ -350,47 +364,42 @@ func (c *Controller) handleUpdateOvnFip(key string) error { klog.Error(err) return err } - if cachedEip.Status.V4Ip == "" { - err := fmt.Errorf("failed to create v4 fip %s, eip %s has no v4 ip", cachedFip.Name, eipName) - klog.Error(err) - return err - } + var v4Eip, v6Eip, v4IP, v6IP string + v4Eip = cachedEip.Status.V4Ip + v6Eip = cachedEip.Status.V6Ip if err = c.isOvnFipDuplicated(key, cachedEip.Spec.V4Ip); err != nil { err = fmt.Errorf("failed to add fip %s, %v", key, err) klog.Error(err) return err } - var mac, internalV4Ip, subnetName, vpcName string - if cachedFip.Spec.Vpc != "" { - vpcName = cachedFip.Spec.Vpc - } - if cachedFip.Spec.V4Ip != "" { - internalV4Ip = cachedFip.Spec.V4Ip - } - if internalV4Ip == "" && cachedFip.Spec.IPName != "" { + var subnetName, vpcName, ipName string + vpcName = cachedFip.Spec.Vpc + v4IP = cachedFip.Spec.V4Ip + v6IP = cachedFip.Spec.V6Ip + ipName = cachedFip.Spec.IPName + if ipName != "" { if cachedFip.Spec.IPType == util.Vip { - internalVip, err := c.virtualIpsLister.Get(cachedFip.Spec.IPName) + internalVip, err := c.virtualIpsLister.Get(ipName) if err != nil { - klog.Errorf("failed to get vip %s, %v", cachedFip.Spec.IPName, err) + klog.Errorf("failed to get vip %s, %v", ipName, err) return err } - internalV4Ip = internalVip.Status.V4ip + v4IP = internalVip.Status.V4ip + v6IP = internalVip.Status.V6ip subnetName = internalVip.Spec.Subnet - // though vip lsp has its mac, vip always use its parent lsp nic mac - // and vip could float to different parent lsp nic + // vip lsp has its mac, but vip always use its parent lsp nic mac + // vip could float to different parent lsp nic // all vip its parent lsp acl should allow the vip ip } else { - internalIP, err := c.ipsLister.Get(cachedFip.Spec.IPName) + internalIP, err := c.ipsLister.Get(ipName) if err != nil { - klog.Errorf("failed to get ip %s, %v", cachedFip.Spec.IPName, err) + klog.Errorf("failed to get ip %s, %v", ipName, err) return err } - internalV4Ip = internalIP.Spec.V4IPAddress + v4IP = internalIP.Spec.V4IPAddress + v6IP = internalIP.Spec.V6IPAddress subnetName = internalIP.Spec.Subnet - mac = internalIP.Spec.MacAddress - // mac is necessary while using distributed router fip, fip use lsp its mac - // centralized router fip not need lsp mac, fip use lrp mac } subnet, err := c.subnetsLister.Get(subnetName) if err != nil { @@ -404,45 +413,35 @@ func (c *Controller) handleUpdateOvnFip(key string) error { return err } } - if internalV4Ip == "" { - err := fmt.Errorf("failed to create v4 fip %s, no internal v4 ip", cachedFip.Name) + if vpcName == "" { + err := fmt.Errorf("failed to update ovn fip %s, no vpc", cachedFip.Name) klog.Error(err) return err } - if vpcName == "" { - err := fmt.Errorf("failed to create v4 fip %s, no vpc", cachedFip.Name) + if vpcName != cachedFip.Status.Vpc { + err := fmt.Errorf("not support change vpc for ovn fip %s", cachedEip.Name) klog.Error(err) return err } - fip := cachedFip.DeepCopy() - // fip change eip - if c.ovnFipChangeEip(fip, cachedEip) { - klog.Infof("fip change ip, old ip '%s', new ip %s", fip.Status.V4Ip, cachedEip.Status.V4Ip) - if err = c.OVNNbClient.DeleteNat(vpcName, ovnnb.NATTypeDNATAndSNAT, fip.Status.V4Ip, internalV4Ip); err != nil { - klog.Errorf("failed to create fip, %v", err) - return err - } - // ovn add fip - options := map[string]string{"staleless": strconv.FormatBool(c.ExternalGatewayType == kubeovnv1.GWDistributedType)} - if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeDNATAndSNAT, cachedEip.Status.V4Ip, - internalV4Ip, mac, cachedFip.Spec.IPName, options); err != nil { - klog.Errorf("failed to create fip, %v", err) - return err - } - if err = c.natLabelAndAnnoOvnEip(eipName, fip.Name, vpcName); err != nil { - klog.Errorf("failed to label fip '%s' in eip %s, %v", fip.Name, eipName, err) - return err - } - if err = c.patchOvnFipAnnotations(key, eipName); err != nil { - klog.Errorf("failed to update label for fip %s, %v", key, err) - return err - } - if err = c.patchOvnFipStatus(key, vpcName, cachedEip.Status.V4Ip, - internalV4Ip, true); err != nil { - klog.Errorf("failed to patch status for fip '%s', %v", key, err) - return err - } - return nil + if v4IP != cachedFip.Status.V4Ip { + err := fmt.Errorf("not support change v4 ip for ovn fip %s", cachedEip.Name) + klog.Error(err) + return err + } + if v6IP != cachedFip.Status.V6Ip { + err := fmt.Errorf("not support change v6 ip for ovn fip %s", cachedEip.Name) + klog.Error(err) + return err + } + if v4Eip != cachedFip.Status.V4Eip { + err := fmt.Errorf("not support change eip for ovn fip %s", cachedEip.Name) + klog.Error(err) + return err + } + if v6Eip != cachedFip.Status.V6Eip { + err := fmt.Errorf("not support change eip for ovn fip %s", cachedEip.Name) + klog.Error(err) + return err } return nil } @@ -457,10 +456,19 @@ func (c *Controller) handleDelOvnFip(key string) error { klog.Error(err) return err } + if cachedFip.Status.Vpc == "" { + return nil + } // ovn delete fip nat - if cachedFip.Status.Vpc != "" && cachedFip.Status.V4Eip != "" && cachedFip.Status.V4Ip != "" { + if cachedFip.Status.V4Eip != "" && cachedFip.Status.V4Ip != "" { if err = c.OVNNbClient.DeleteNat(cachedFip.Status.Vpc, ovnnb.NATTypeDNATAndSNAT, cachedFip.Status.V4Eip, cachedFip.Status.V4Ip); err != nil { - klog.Errorf("failed to delete fip %s, %v", key, err) + klog.Errorf("failed to delete v4 fip %s, %v", key, err) + return err + } + } + if cachedFip.Status.V6Eip != "" && cachedFip.Status.V6Ip != "" { + if err = c.OVNNbClient.DeleteNat(cachedFip.Status.Vpc, ovnnb.NATTypeDNATAndSNAT, cachedFip.Status.V6Eip, cachedFip.Status.V6Ip); err != nil { + klog.Errorf("failed to delete v6 fip %s, %v", key, err) return err } } @@ -503,8 +511,8 @@ func (c *Controller) patchOvnFipAnnotations(key, eipName string) error { patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/annotations", "value": %s }]` raw, _ := json.Marshal(fip.Annotations) patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - _, err := c.config.KubeOvnClient.KubeovnV1().OvnFips().Patch(context.Background(), fip.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}) - if err != nil { + if _, err := c.config.KubeOvnClient.KubeovnV1().OvnFips().Patch(context.Background(), fip.Name, + types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}); err != nil { klog.Errorf("failed to patch annotation for ovn fip %s, %v", fip.Name, err) return err } @@ -577,25 +585,16 @@ func (c *Controller) patchOvnFipStatus(key, vpcName, v4Eip, podIP string, ready return nil } -func (c *Controller) ovnFipChangeEip(fip *kubeovnv1.OvnFip, eip *kubeovnv1.OvnEip) bool { - if fip.Status.V4Ip == "" || eip.Status.V4Ip == "" { - // eip created but not ready - return false - } - if fip.Status.V4Ip != eip.Status.V4Ip { - return true - } - return false -} - func (c *Controller) GetOvnEip(eipName string) (*kubeovnv1.OvnEip, error) { cachedEip, err := c.ovnEipsLister.Get(eipName) if err != nil { klog.Errorf("failed to get eip %s, %v", eipName, err) return nil, err } - if cachedEip.Status.V4Ip == "" { - return nil, fmt.Errorf("eip '%s' is not ready, has no v4ip", eipName) + if cachedEip.Status.V4Ip == "" && cachedEip.Status.V6Ip == "" { + err := fmt.Errorf("eip '%s' is not ready, has no ip", eipName) + klog.Error(err) + return nil, err } return cachedEip, nil } diff --git a/pkg/controller/ovn_snat.go b/pkg/controller/ovn_snat.go index 8d017869973..4d7b1383235 100644 --- a/pkg/controller/ovn_snat.go +++ b/pkg/controller/ovn_snat.go @@ -182,7 +182,7 @@ func (c *Controller) handleAddOvnSnatRule(key string) error { klog.Error(err) return err } - if cachedSnat.Status.Ready && cachedSnat.Status.V4IpCidr != "" { + if cachedSnat.Status.Ready { // already ok return nil } @@ -205,31 +205,27 @@ func (c *Controller) handleAddOvnSnatRule(key string) error { klog.Error(err) return err } - if cachedEip.Status.V4Ip == "" { - err := fmt.Errorf("failed to create v4 snat %s, eip %s has no v4 ip", cachedSnat.Name, eipName) - klog.Error(err) - return err - } - var v4IpCidr, vpcName string - if cachedSnat.Spec.Vpc != "" { - vpcName = cachedSnat.Spec.Vpc - } - if cachedSnat.Spec.V4IpCidr != "" { - v4IpCidr = cachedSnat.Spec.V4IpCidr - } - if v4IpCidr == "" && cachedSnat.Spec.VpcSubnet != "" { - subnet, err := c.subnetsLister.Get(cachedSnat.Spec.VpcSubnet) + var v4Eip, v6Eip, v4IpCidr, v6IpCidr, vpcName, subnetName, ipName string + v4Eip = cachedEip.Status.V4Ip + v6Eip = cachedEip.Status.V6Ip + v4IpCidr = cachedSnat.Spec.V4IpCidr + v6IpCidr = cachedSnat.Spec.V6IpCidr + vpcName = cachedSnat.Spec.Vpc + subnetName = cachedSnat.Spec.VpcSubnet + if v4IpCidr == "" && subnetName != "" { + subnet, err := c.subnetsLister.Get(subnetName) if err != nil { - klog.Errorf("failed to get vpc subnet %s, %v", cachedSnat.Spec.VpcSubnet, err) + klog.Errorf("failed to get vpc subnet %s, %v", subnetName, err) return err } vpcName = subnet.Spec.Vpc - v4IpCidr = subnet.Spec.CIDRBlock + v4IpCidr, v6IpCidr = util.SplitStringIP(subnet.Spec.CIDRBlock) } - if v4IpCidr == "" && cachedSnat.Spec.IPName != "" { - vpcPodIP, err := c.ipsLister.Get(cachedSnat.Spec.IPName) + ipName = cachedSnat.Spec.IPName + if v4IpCidr == "" && ipName != "" { + vpcPodIP, err := c.ipsLister.Get(ipName) if err != nil { - klog.Errorf("failed to get pod ip %s, %v", cachedSnat.Spec.IPName, err) + klog.Errorf("failed to get pod ip %s, %v", ipName, err) return err } subnet, err := c.subnetsLister.Get(vpcPodIP.Spec.Subnet) @@ -239,24 +235,36 @@ func (c *Controller) handleAddOvnSnatRule(key string) error { } vpcName = subnet.Spec.Vpc v4IpCidr = vpcPodIP.Spec.V4IPAddress + v6IpCidr = vpcPodIP.Spec.V6IPAddress } - if v4IpCidr == "" { - // only support IPv4 snat - err = fmt.Errorf("failed to get v4 internal ip for snat %s", key) + if vpcName == "" { + err := fmt.Errorf("failed to get vpc for snat %s", cachedSnat.Name) klog.Error(err) return err } - if vpcName == "" { - err := fmt.Errorf("failed to get vpc for snat %s", cachedSnat.Name) + if v4IpCidr == "" && v6IpCidr == "" { + err = fmt.Errorf("failed to get subnet or ip cidr for snat %s", key) klog.Error(err) return err } - - // about conflicts: if multi vpc snat use the same eip, if only one gw node exist, it may should work - if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeSNAT, cachedEip.Spec.V4Ip, v4IpCidr, "", "", nil); err != nil { - klog.Errorf("failed to create snat, %v", err) + if v4Eip == "" && v6Eip == "" { + err := fmt.Errorf("failed to create snat %s, no external ip", cachedSnat.Name) + klog.Error(err) return err } + // about conflicts: if multi vpc snat use the same eip, if only one gw node exist, it may should work + if v4IpCidr != "" && v4Eip != "" { + if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeSNAT, v4Eip, v4IpCidr, "", "", nil); err != nil { + klog.Errorf("failed to create v4 snat, %v", err) + return err + } + } + if v6IpCidr != "" && v6Eip != "" { + if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeSNAT, v6Eip, v6IpCidr, "", "", nil); err != nil { + klog.Errorf("failed to create v6 snat, %v", err) + return err + } + } if err := c.handleAddOvnSnatFinalizer(cachedSnat); err != nil { klog.Errorf("failed to add finalizer for ovn snat %s, %v", cachedSnat.Name, err) return err @@ -269,7 +277,7 @@ func (c *Controller) handleAddOvnSnatRule(key string) error { klog.Errorf("failed to patch label for snat %s, %v", key, err) return err } - if err = c.patchOvnSnatStatus(key, vpcName, cachedEip.Spec.V4Ip, v4IpCidr, true); err != nil { + if err = c.patchOvnSnatStatus(key, vpcName, v4Eip, v6Eip, v4IpCidr, v6IpCidr, true); err != nil { klog.Errorf("failed to update status for snat %s, %v", key, err) return err } @@ -289,16 +297,8 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error { klog.Error(err) return err } - // should delete - if !cachedSnat.DeletionTimestamp.IsZero() { - klog.Infof("ovn delete snat %s", key) - if cachedSnat.Status.Vpc != "" && cachedSnat.Status.V4Eip != "" && cachedSnat.Status.V4IpCidr != "" { - if err = c.OVNNbClient.DeleteNat(cachedSnat.Status.Vpc, ovnnb.NATTypeSNAT, cachedSnat.Status.V4Eip, cachedSnat.Status.V4IpCidr); err != nil { - klog.Errorf("failed to delete snat, %v", err) - return err - } - } - c.resetOvnEipQueue.Add(cachedSnat.Spec.OvnEip) + if !cachedSnat.Status.Ready { + klog.Infof("wait ovn snat %s to be ready only in the handle add process", cachedSnat.Name) return nil } klog.Infof("handle update ovn snat %s", key) @@ -320,31 +320,27 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error { klog.Error(err) return err } - if cachedEip.Status.V4Ip == "" { - err := fmt.Errorf("failed to create v4 snat %s, eip %s has no v4 ip", cachedSnat.Name, eipName) - klog.Error(err) - return err - } - var v4IpCidr, vpcName string - if cachedSnat.Spec.Vpc != "" { - vpcName = cachedSnat.Spec.Vpc - } - if cachedSnat.Spec.V4IpCidr != "" { - v4IpCidr = cachedSnat.Spec.V4IpCidr - } - if v4IpCidr == "" && cachedSnat.Spec.VpcSubnet != "" { - subnet, err := c.subnetsLister.Get(cachedSnat.Spec.VpcSubnet) + var v4Eip, v6Eip, v4IpCidr, v6IpCidr, vpcName, subnetName, ipName string + v4Eip = cachedEip.Status.V4Ip + v6Eip = cachedEip.Status.V6Ip + v4IpCidr = cachedSnat.Spec.V4IpCidr + v6IpCidr = cachedSnat.Spec.V6IpCidr + vpcName = cachedSnat.Spec.Vpc + subnetName = cachedSnat.Spec.VpcSubnet + if v4IpCidr == "" && subnetName != "" { + subnet, err := c.subnetsLister.Get(subnetName) if err != nil { - klog.Errorf("failed to get vpc subnet %s, %v", cachedSnat.Spec.VpcSubnet, err) + klog.Errorf("failed to get vpc subnet %s, %v", subnetName, err) return err } vpcName = subnet.Spec.Vpc - v4IpCidr = subnet.Spec.CIDRBlock + v4IpCidr, v6IpCidr = util.SplitStringIP(subnet.Spec.CIDRBlock) } - if v4IpCidr == "" && cachedSnat.Spec.IPName != "" { - vpcPodIP, err := c.ipsLister.Get(cachedSnat.Spec.IPName) + ipName = cachedSnat.Spec.IPName + if v4IpCidr == "" && ipName != "" { + vpcPodIP, err := c.ipsLister.Get(ipName) if err != nil { - klog.Errorf("failed to get pod ip %s, %v", cachedSnat.Spec.IPName, err) + klog.Errorf("failed to get pod ip %s, %v", ipName, err) return err } subnet, err := c.subnetsLister.Get(vpcPodIP.Spec.Subnet) @@ -354,49 +350,47 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error { } vpcName = subnet.Spec.Vpc v4IpCidr = vpcPodIP.Spec.V4IPAddress + v6IpCidr = vpcPodIP.Spec.V6IPAddress } - if v4IpCidr == "" { - // only support IPv4 snat - err = fmt.Errorf("failed to get v4 internal ip for snat %s", key) + if vpcName == "" { + err := fmt.Errorf("failed to get vpc for snat %s", cachedSnat.Name) klog.Error(err) return err } - if vpcName == "" { - err := fmt.Errorf("failed to get vpc for snat %s", cachedSnat.Name) + if vpcName != cachedSnat.Status.Vpc { + err := fmt.Errorf("failed to update snat %s, vpc changed", key) klog.Error(err) return err } - if cachedEip.Spec.Type == util.OvnEipTypeLSP { - // eip is using by ecmp nexthop lsp, nat can not use - err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.OvnEipTypeLSP, eipName) + if v4IpCidr == "" && v6IpCidr == "" { + err = fmt.Errorf("failed to get subnet or ip cidr for snat %s", key) klog.Error(err) return err } - // snat change eip - if c.ovnSnatChangeEip(cachedSnat, cachedEip) { - klog.Infof("snat change ip, old ip %s, new ip %s", cachedEip.Status.V4Ip, cachedEip.Spec.V4Ip) - if err = c.OVNNbClient.DeleteNat(vpcName, ovnnb.NATTypeSNAT, cachedEip.Status.V4Ip, v4IpCidr); err != nil { - klog.Errorf("failed to delte snat, %v", err) - return err - } - // ovn add snat with new eip - if err = c.OVNNbClient.AddNat(vpcName, ovnnb.NATTypeSNAT, cachedEip.Spec.V4Ip, v4IpCidr, "", "", nil); err != nil { - klog.Errorf("failed to create snat, %v", err) - return err - } - if err = c.natLabelAndAnnoOvnEip(eipName, cachedSnat.Name, vpcName); err != nil { - klog.Errorf("failed to label snat '%s' in eip %s, %v", cachedSnat.Name, eipName, err) - return err - } - if err = c.patchOvnSnatAnnotation(key, eipName); err != nil { - klog.Errorf("failed to patch label for snat %s, %v", key, err) - return err - } - if err = c.patchOvnSnatStatus(key, vpcName, cachedEip.Spec.V4Ip, v4IpCidr, true); err != nil { - klog.Errorf("failed to update status for snat %s, %v", key, err) - return err - } - return nil + if v4Eip == "" && v6Eip == "" { + err := fmt.Errorf("failed to update snat %s, no external ip", cachedSnat.Name) + klog.Error(err) + return err + } + if v4Eip != cachedSnat.Status.V4Eip { + err := fmt.Errorf("failed to update snat %s, v4 eip changed", key) + klog.Error(err) + return err + } + if v6Eip != cachedSnat.Status.V6Eip { + err := fmt.Errorf("failed to update snat %s, v6 eip changed", key) + klog.Error(err) + return err + } + if v4IpCidr != cachedSnat.Status.V4IpCidr { + err := fmt.Errorf("failed to update snat %s, v4 ip cidr changed", key) + klog.Error(err) + return err + } + if v6IpCidr != cachedSnat.Status.V6IpCidr { + err := fmt.Errorf("failed to update snat %s, v6 ip cidr changed", key) + klog.Error(err) + return err } return nil } @@ -415,10 +409,17 @@ func (c *Controller) handleDelOvnSnatRule(key string) error { if cachedSnat.Status.Vpc != "" && cachedSnat.Status.V4Eip != "" && cachedSnat.Status.V4IpCidr != "" { if err = c.OVNNbClient.DeleteNat(cachedSnat.Status.Vpc, ovnnb.NATTypeSNAT, cachedSnat.Status.V4Eip, cachedSnat.Status.V4IpCidr); err != nil { - klog.Errorf("failed to delete snat %s, %v", key, err) + klog.Errorf("failed to delete v4 snat %s, %v", key, err) return err } } + if cachedSnat.Status.Vpc != "" && cachedSnat.Status.V6Eip != "" && cachedSnat.Status.V6IpCidr != "" { + if err = c.OVNNbClient.DeleteNat(cachedSnat.Status.Vpc, ovnnb.NATTypeSNAT, cachedSnat.Status.V6Eip, cachedSnat.Status.V6IpCidr); err != nil { + klog.Errorf("failed to delete v6 snat, %v", err) + return err + } + } + c.resetOvnEipQueue.Add(cachedSnat.Spec.OvnEip) if err = c.handleDelOvnSnatFinalizer(cachedSnat); err != nil { klog.Errorf("failed to remove finalizer for ovn snat %s, %v", cachedSnat.Name, err) return err @@ -429,7 +430,7 @@ func (c *Controller) handleDelOvnSnatRule(key string) error { return nil } -func (c *Controller) patchOvnSnatStatus(key, vpc, v4Eip, v4IpCidr string, ready bool) error { +func (c *Controller) patchOvnSnatStatus(key, vpc, v4Eip, v6Eip, v4IpCidr, v6IpCidr string, ready bool) error { oriSnat, err := c.ovnSnatRulesLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { @@ -446,11 +447,13 @@ func (c *Controller) patchOvnSnatStatus(key, vpc, v4Eip, v4IpCidr string, ready needUpdateLabel = true snat.Labels = map[string]string{ util.EipV4IpLabel: v4Eip, + util.EipV6IpLabel: v6Eip, } } else if snat.Labels[util.EipV4IpLabel] != v4Eip { op = "replace" needUpdateLabel = true snat.Labels[util.EipV4IpLabel] = v4Eip + snat.Labels[util.EipV6IpLabel] = v4Eip } if needUpdateLabel { patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]` @@ -475,10 +478,18 @@ func (c *Controller) patchOvnSnatStatus(key, vpc, v4Eip, v4IpCidr string, ready snat.Status.V4Eip = v4Eip changed = true } + if v6Eip != "" && snat.Status.V6Eip != v6Eip { + snat.Status.V6Eip = v6Eip + changed = true + } if v4IpCidr != "" && snat.Status.V4IpCidr != v4IpCidr { snat.Status.V4IpCidr = v4IpCidr changed = true } + if v6IpCidr != "" && snat.Status.V6IpCidr != v6IpCidr { + snat.Status.V6IpCidr = v6IpCidr + changed = true + } if changed { bytes, err := snat.Status.Bytes() if err != nil { @@ -531,17 +542,6 @@ func (c *Controller) patchOvnSnatAnnotation(key, eipName string) error { return nil } -func (c *Controller) ovnSnatChangeEip(snat *kubeovnv1.OvnSnatRule, eip *kubeovnv1.OvnEip) bool { - if snat.Status.V4Eip == "" || eip.Spec.V4Ip == "" { - // eip created but not ready - return false - } - if snat.Status.V4Eip != eip.Spec.V4Ip { - return true - } - return false -} - func (c *Controller) syncOvnSnatFinalizer(cl client.Client) error { // migrate depreciated finalizer to new finalizer rules := &kubeovnv1.OvnSnatRuleList{} diff --git a/pkg/util/const.go b/pkg/util/const.go index c929799901b..4d3ff8c895b 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -44,6 +44,7 @@ const ( VpcNatAnnotation = "ovn.kubernetes.io/vpc_nat" OvnEipTypeLabel = "ovn.kubernetes.io/ovn_eip_type" EipV4IpLabel = "ovn.kubernetes.io/eip_v4_ip" + EipV6IpLabel = "ovn.kubernetes.io/eip_v6_ip" SwitchLBRuleVipsAnnotation = "ovn.kubernetes.io/switch_lb_vip" SwitchLBRuleVip = "switch_lb_vip" diff --git a/pkg/webhook/ovn_nat_gateway.go b/pkg/webhook/ovn_nat_gateway.go index 4faafd45030..712aae28aeb 100644 --- a/pkg/webhook/ovn_nat_gateway.go +++ b/pkg/webhook/ovn_nat_gateway.go @@ -62,12 +62,12 @@ func (v *ValidatingHook) ovnEipUpdateHook(ctx context.Context, req admission.Req return ctrlwebhook.Allowed("by pass") } -func (v *ValidatingHook) isOvnEipInUse(ctx context.Context, eipV4IP string) (string, error) { +func (v *ValidatingHook) isOvnEipInUse(ctx context.Context, eipV4IP, eipV6IP string) (string, error) { var err error dnatList := ovnv1.OvnDnatRuleList{} fipList := ovnv1.OvnFipList{} snatList := ovnv1.OvnSnatRuleList{} - opts := cli.MatchingLabels{util.EipV4IpLabel: eipV4IP} + opts := cli.MatchingLabels{util.EipV4IpLabel: eipV4IP, util.EipV6IpLabel: eipV6IP} err = v.cache.List(ctx, &dnatList, opts) if err != nil { klog.Errorf("failed to list ovn dnat, %v", err) @@ -103,7 +103,7 @@ func (v *ValidatingHook) ovnEipDeleteHook(ctx context.Context, req admission.Req if eip.Status.Ready { var err error - nat, err := v.isOvnEipInUse(ctx, eip.Spec.V4Ip) + nat, err := v.isOvnEipInUse(ctx, eip.Spec.V4Ip, eip.Spec.V6Ip) if nat != "" { err = fmt.Errorf("OvnEip %s is still using by ovn nat", eip.Name) return ctrlwebhook.Errored(http.StatusBadRequest, err) @@ -268,8 +268,8 @@ func (v *ValidatingHook) ValidateOvnDnat(ctx context.Context, dnat *ovnv1.OvnDna err := fmt.Errorf("should set spec ovnEip") return err } - if dnat.Spec.IPName == "" && dnat.Spec.V4Ip == "" { - err := fmt.Errorf("should set spec ipName or v4Ip") + if dnat.Spec.IPName == "" && dnat.Spec.V4Ip == "" && dnat.Spec.V6Ip == "" { + err := fmt.Errorf("should set spec ipName or v4 or v6 Ip") return err } eip := &ovnv1.OvnEip{} @@ -324,18 +324,18 @@ func (v *ValidatingHook) ValidateOvnSnat(ctx context.Context, snat *ovnv1.OvnSna return err } - if snat.Spec.Vpc != "" && snat.Spec.V4IpCidr == "" { - err := fmt.Errorf("should set spec v4IpCidr (subnet cidr or ip address) when spec vpc is set") + if snat.Spec.Vpc != "" && snat.Spec.V4IpCidr == "" && snat.Spec.V6IpCidr == "" { + err := fmt.Errorf("should set spec v4 or v6 IpCidr (subnet cidr or ip address) when spec vpc is set") return err } - if snat.Spec.Vpc == "" && snat.Spec.V4IpCidr != "" { - err := fmt.Errorf("should set spec vpc when spec v4IpCidr is set") + if snat.Spec.Vpc == "" && snat.Spec.V4IpCidr != "" && snat.Spec.V6IpCidr != "" { + err := fmt.Errorf("should set spec vpc while spec v4 or v6 IpCidr is not set") return err } - if snat.Spec.VpcSubnet == "" && snat.Spec.IPName == "" && snat.Spec.Vpc == "" && snat.Spec.V4IpCidr == "" { - err := fmt.Errorf("should set spec vpcSubnet or ipName or vpc and v4IpCidr at least") + if snat.Spec.VpcSubnet == "" && snat.Spec.IPName == "" && snat.Spec.Vpc == "" && snat.Spec.V4IpCidr == "" && snat.Spec.V6IpCidr == "" { + err := fmt.Errorf("should set spec vpcSubnet or ipName or vpc and v4 and v6 IpCidr at least") return err } @@ -349,8 +349,8 @@ func (v *ValidatingHook) ValidateOvnFip(ctx context.Context, fip *ovnv1.OvnFip) err := fmt.Errorf("should set spec ovnEip") return err } - if fip.Spec.IPName == "" && fip.Spec.V4Ip == "" { - err := fmt.Errorf("should set spec ipName or v4Ip") + if fip.Spec.IPName == "" && fip.Spec.V4Ip == "" && fip.Spec.V6Ip == "" { + err := fmt.Errorf("should set spec ipName or ip") return err } eip := &ovnv1.OvnEip{} diff --git a/yamls/crd.yaml b/yamls/crd.yaml index a305a378a51..d3e91e68db8 100644 --- a/yamls/crd.yaml +++ b/yamls/crd.yaml @@ -1268,9 +1268,15 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string + - jsonPath: .status.v6Eip + name: V6Eip + type: string - jsonPath: .status.v4Ip name: V4Ip type: string + - jsonPath: .status.v6Ip + name: V6Ip + type: string - jsonPath: .status.ready name: Ready type: boolean @@ -1325,6 +1331,8 @@ spec: type: string v4Ip: type: string + v6Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1353,9 +1361,15 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string + - jsonPath: .status.v6Eip + name: V6Eip + type: string - jsonPath: .status.v4IpCidr name: V4IpCidr type: string + - jsonPath: .status.v6IpCidr + name: V6IpCidr + type: string - jsonPath: .status.ready name: Ready type: boolean @@ -1370,8 +1384,12 @@ spec: type: boolean v4Eip: type: string + v6Eip: + type: string v4IpCidr: type: string + v6IpCidr: + type: string vpc: type: string conditions: @@ -1404,6 +1422,8 @@ spec: type: string v4IpCidr: type: string + v6IpCidr: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1438,9 +1458,15 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string + - jsonPath: .status.v6Eip + name: V6Eip + type: string - jsonPath: .status.v4Ip name: V4Ip type: string + - jsonPath: .status.v6Ip + name: V6Ip + type: string - jsonPath: .status.internalPort name: InternalPort type: string @@ -1512,6 +1538,8 @@ spec: type: string v4Ip: type: string + v6Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition