Skip to content

Commit

Permalink
简化 ovn eip 类型 (#3107)
Browse files Browse the repository at this point in the history 
* 简化 ovn eip 类型
* support nat reuse lrp ip
* fix const value
* fix ecmp bfd static route
* fix cleanup
  • Loading branch information
@bobz965
bobz965 committed Aug 4, 2023
1 parent a0c5e38 commit 6ba997d
Show file tree
Hide file tree
Showing 13 changed files with 132 additions and 122 deletions.
20 changes: 15 additions & 5 deletions dist/images/cleanup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ for vd in $(kubectl get vpc-dns -o name); do
kubectl delete --ignore-not-found $vd
done

for ip in $(kubectl get ip -o name); do
kubectl delete --ignore-not-found $ip
done

for vip in $(kubectl get vip -o name); do
kubectl delete --ignore-not-found $vip
done
Expand All @@ -38,6 +42,10 @@ for eip in $(kubectl get eip -o name); do
kubectl delete --ignore-not-found $eip
done

for odnat in $(kubectl get odnat -o name); do
kubectl delete --ignore-not-found $odnat
done

for osnat in $(kubectl get osnat -o name); do
kubectl delete --ignore-not-found $osnat
done
Expand All @@ -63,6 +71,8 @@ for subnet in $(kubectl get subnet -o name); do
kubectl patch "$subnet" --type='json' -p '[{"op": "replace", "path": "/metadata/finalizers", "value": []}]'
kubectl delete --ignore-not-found "$subnet"
done
# subnet join will recreate, so delete subnet crd right now
kubectl delete --ignore-not-found crd subnets.kubeovn.io
set -e

for vpc in $(kubectl get vpc -o name); do
Expand Down Expand Up @@ -120,21 +130,21 @@ kubectl delete --ignore-not-found crd \
security-groups.kubeovn.io \
ips.kubeovn.io \
ippools.kubeovn.io \
subnets.kubeovn.io \
vpc-nat-gateways.kubeovn.io \
vpcs.kubeovn.io \
vlans.kubeovn.io \
provider-networks.kubeovn.io \
iptables-dnat-rules.kubeovn.io \
iptables-eips.kubeovn.io \
iptables-fip-rules.kubeovn.io \
iptables-snat-rules.kubeovn.io \
iptables-fip-rules.kubeovn.io \
iptables-eips.kubeovn.io \
vips.kubeovn.io \
switch-lb-rules.kubeovn.io \
vpc-dnses.kubeovn.io \
ovn-eips.kubeovn.io ovn-fips.kubeovn.io \
ovn-snat-rules.kubeovn.io \
ovn-dnat-rules.kubeovn.io \
ovn-snat-rules.kubeovn.io \
ovn-fips.kubeovn.io \
ovn-eips.kubeovn.io \
qos-policies.kubeovn.io

# Remove annotations/labels in namespaces and nodes
Expand Down
8 changes: 4 additions & 4 deletions pkg/apis/kubeovn/v1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -949,10 +949,10 @@ type OvnEipSpec struct {
V6Ip string `json:"v6Ip"`
MacAddress string `json:"macAddress"`
Type string `json:"type"`
// usage type: eip, lrp, node external gw
// eip: only used by nat, fip, snat, dnat, all the nat type will record int the eip status
// lrp: logical router port
// node external gw: is lsp, in the case of bfd session between lrp and lsp, the lsp is on the node as external gateway
// usage type: lrp, lsp, nat
// nat: used by nat: dnat, snat, fip
// lrp: lrp created by vpc enable external, and also could be used by nat
// lsp: in the case of bfd session between lrp and lsp, the lsp is on the node as ecmp nexthop
}

// OvnEipCondition describes the state of an object at a certain point.
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/external-gw.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ func (c *Controller) createDefaultVpcLrpEip(config map[string]string) (string, s
klog.Errorf("failed to acquire ip address for default vpc lrp %s, %v", lrpEipName, err)
return "", "", err
}
if err := c.createOrUpdateCrdOvnEip(lrpEipName, c.config.ExternalGatewaySwitch, v4ip, v6ip, mac, util.LrpUsingEip); err != nil {
if err := c.createOrUpdateCrdOvnEip(lrpEipName, c.config.ExternalGatewaySwitch, v4ip, v6ip, mac, util.Lrp); err != nil {
klog.Errorf("failed to create ovn eip cr for lrp %s, %v", lrpEipName, err)
return "", "", err
}
Expand Down
24 changes: 10 additions & 14 deletions pkg/controller/ovn_dnat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ func (c *Controller) handleAddOvnDnatRule(key string) error {
klog.V(3).Infof("handle add dnat %s", key)

var internalV4Ip, mac, subnetName string
if cachedDnat.Spec.IpType == util.NatUsingVip {
if cachedDnat.Spec.IpType == util.Vip {
internalVip, err := c.virtualIpsLister.Get(cachedDnat.Spec.IpName)
if err != nil {
klog.Errorf("failed to get vip %s, %v", cachedDnat.Spec.IpName, err)
Expand Down Expand Up @@ -248,8 +248,10 @@ func (c *Controller) handleAddOvnDnatRule(key string) error {
return err
}

if cachedEip.Status.Type != "" && cachedEip.Status.Type != util.NatUsingEip {
err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip)
if cachedEip.Spec.Type == util.Lsp {
// eip is using by ecmp nexthop lsp, nat can not use
err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName)
klog.Error(err)
return err
}

Expand Down Expand Up @@ -354,7 +356,7 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error {

klog.V(3).Infof("handle update dnat %s", key)
var internalV4Ip, mac, subnetName string
if cachedDnat.Spec.IpType == util.NatUsingVip {
if cachedDnat.Spec.IpType == util.Vip {
internalVip, err := c.virtualIpsLister.Get(cachedDnat.Spec.IpName)
if err != nil {
klog.Errorf("failed to get vip %s, %v", cachedDnat.Spec.IpName, err)
Expand Down Expand Up @@ -410,16 +412,10 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error {
return err
}

if cachedEip.Spec.Type != "" && cachedEip.Spec.Type != util.DnatUsingEip {
// eip is in use by other nat
err = fmt.Errorf("failed to update dnat %s, eip '%s' is using by %s", key, eipName, cachedEip.Spec.Type)
return err
}

if cachedEip.Spec.Type == util.DnatUsingEip &&
cachedEip.Annotations[util.VpcNatAnnotation] != "" &&
cachedEip.Annotations[util.VpcNatAnnotation] != cachedDnat.Name {
err = fmt.Errorf("failed to update dnat %s, eip '%s' is using by other dnat %s", key, eipName, cachedEip.Annotations[util.VpcNatAnnotation])
if cachedEip.Spec.Type == util.Lsp {
// eip is using by ecmp nexthop lsp, nat can not use
err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName)
klog.Error(err)
return err
}

Expand Down
14 changes: 7 additions & 7 deletions pkg/controller/ovn_eip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ func (c *Controller) handleAddOvnEip(key string) error {
return err
}

if cachedEip.Spec.Type == util.NodeExtGwUsingEip {
if cachedEip.Spec.Type == util.Lsp {
mergedIp := util.GetStringIP(v4ip, v6ip)
if err := c.ovnClient.CreateBareLogicalSwitchPort(subnet.Name, portName, mergedIp, mac); err != nil {
klog.Error("failed to create lsp for ovn eip %s, %v", key, err)
Expand All @@ -262,8 +262,8 @@ func (c *Controller) handleAddOvnEip(key string) error {
klog.Errorf("failed to create or update ovn eip '%s', %v", cachedEip.Name, err)
return err
}
if cachedEip.Spec.Type != util.NodeExtGwUsingEip {
// node ext gw eip has a nic on node, so left node to make it ready
if cachedEip.Spec.Type != util.Lsp {
// node ext gw use lsp eip, has a nic on gw node, so left node to make it ready
if err = c.patchOvnEipStatus(key, true); err != nil {
klog.Errorf("failed to patch ovn eip %s: %v", key, err)
return err
Expand Down Expand Up @@ -302,8 +302,8 @@ func (c *Controller) handleUpdateOvnEip(key string) error {
}
return nil
}
if cachedEip.Spec.Type != util.NodeExtGwUsingEip {
// node ext gw eip has a nic on node, so left node to make it ready
if cachedEip.Spec.Type != util.Lsp {
// node ext gw use lsp eip, has a nic on gw node, so left node to make it ready
if err = c.patchOvnEipStatus(key, true); err != nil {
klog.Errorf("failed to patch ovn eip %s: %v", key, err)
return err
Expand Down Expand Up @@ -345,14 +345,14 @@ func (c *Controller) handleDelOvnEip(eip *kubeovnv1.OvnEip) error {
return err
}

if eip.Spec.Type == util.NodeExtGwUsingEip {
if eip.Spec.Type == util.Lsp {
if err := c.ovnClient.DeleteLogicalSwitchPort(eip.Name); err != nil {
klog.Errorf("failed to delete lsp %s, %v", eip.Name, err)
return err
}
}

if eip.Spec.Type == util.LrpUsingEip {
if eip.Spec.Type == util.Lrp {
if err := c.ovnClient.DeleteLogicalRouterPort(eip.Name); err != nil {
klog.Errorf("failed to delete lrp %s, %v", eip.Name, err)
return err
Expand Down
18 changes: 14 additions & 4 deletions pkg/controller/ovn_fip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ func (c *Controller) handleAddOvnFip(key string) error {
}
klog.V(3).Infof("handle add fip %s", key)
var internalV4Ip, mac, subnetName string
if cachedFip.Spec.IpType == util.NatUsingVip {
if cachedFip.Spec.IpType == util.Vip {
internalVip, err := c.virtualIpsLister.Get(cachedFip.Spec.IpName)
if err != nil {
klog.Errorf("failed to get vip %s, %v", cachedFip.Spec.IpName, err)
Expand Down Expand Up @@ -237,6 +237,13 @@ func (c *Controller) handleAddOvnFip(key string) error {
return err
}

if cachedEip.Spec.Type == util.Lsp {
// eip is using by ecmp nexthop lsp, nat can not use
err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName)
klog.Error(err)
return err
}

if err = c.ovnFipTryUseEip(key, cachedEip.Spec.V4Ip); err != nil {
err = fmt.Errorf("failed to add fip %s, %v", key, err)
klog.Error(err)
Expand All @@ -258,6 +265,7 @@ func (c *Controller) handleAddOvnFip(key string) error {
err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip)
return err
}

if err = c.ovnFipTryUseEip(key, cachedEip.Spec.V4Ip); err != nil {
err = fmt.Errorf("failed to update fip %s, %v", key, err)
klog.Error(err)
Expand Down Expand Up @@ -310,7 +318,7 @@ func (c *Controller) handleUpdateOvnFip(key string) error {
}
klog.V(3).Infof("handle update fip %s", key)
var internalV4Ip, mac, subnetName string
if cachedFip.Spec.IpType == util.NatUsingVip {
if cachedFip.Spec.IpType == util.Vip {
internalVip, err := c.virtualIpsLister.Get(cachedFip.Spec.IpName)
if err != nil {
klog.Errorf("failed to get vip %s, %v", cachedFip.Spec.IpName, err)
Expand Down Expand Up @@ -341,8 +349,10 @@ func (c *Controller) handleUpdateOvnFip(key string) error {
klog.Errorf("failed to get eip, %v", err)
return err
}
if cachedEip.Status.Type != "" && cachedEip.Status.Type != util.NatUsingEip {
err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip)
if cachedEip.Spec.Type == util.Lsp {
// eip is using by ecmp nexthop lsp, nat can not use
err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName)
klog.Error(err)
return err
}
if err = c.ovnFipTryUseEip(key, cachedEip.Spec.V4Ip); err != nil {
Expand Down
12 changes: 8 additions & 4 deletions pkg/controller/ovn_snat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,8 +197,10 @@ func (c *Controller) handleAddOvnSnatRule(key string) error {
return err
}

if cachedEip.Status.Type != "" && cachedEip.Status.Type != util.NatUsingEip {
err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip)
if cachedEip.Spec.Type == util.Lsp {
// eip is using by ecmp nexthop lsp, nat can not use
err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName)
klog.Error(err)
return err
}

Expand Down Expand Up @@ -303,8 +305,10 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error {
return nil
}

if cachedEip.Status.Type != "" && cachedEip.Status.Type != util.NatUsingEip {
err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip)
if cachedEip.Spec.Type == util.Lsp {
// eip is using by ecmp nexthop lsp, nat can not use
err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName)
klog.Error(err)
return err
}

Expand Down
32 changes: 14 additions & 18 deletions pkg/controller/subnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1190,13 +1190,13 @@ func (c *Controller) reconcileNamespaces(subnet *kubeovnv1.Subnet) error {
func (c *Controller) reconcileCustomVpcBfdStaticRoute(vpcName, subnetName string) error {
// vpc enable bfd and subnet enable ecmp
// use static ecmp route with bfd
ovnEips, err := c.ovnEipsLister.List(labels.SelectorFromSet(labels.Set{util.OvnEipTypeLabel: util.NodeExtGwUsingEip}))
ovnEips, err := c.ovnEipsLister.List(labels.SelectorFromSet(labels.Set{util.OvnEipTypeLabel: util.Lsp}))
if err != nil {
klog.Errorf("failed to list node external ovn eip, %v", err)
return err
}
if len(ovnEips) < 2 {
err := fmt.Errorf("ecmp route with bfd for HA, which need two %s type eips at least, has %d", util.NodeExtGwUsingEip, len(ovnEips))
err := fmt.Errorf("ecmp route with bfd for HA, which need two %s type eips at least, has %d", util.Lsp, len(ovnEips))
klog.Error(err)
return err
}
Expand Down Expand Up @@ -1266,17 +1266,15 @@ func (c *Controller) reconcileCustomVpcBfdStaticRoute(vpcName, subnetName string
}
}
if needUpdate {
if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}); err != nil {
if vpc, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}); err != nil {
klog.Errorf("failed to update vpc spec static route %s, %v", vpc.Name, err)
return err
}
if err = c.patchVpcBfdStatus(vpc.Name); err != nil {
klog.Errorf("failed to patch vpc %s, %v", vpc.Name, err)
return err
}
}

if err = c.patchVpcBfdStatus(vpc.Name); err != nil {
klog.Errorf("failed to patch vpc %s, %v", vpc.Name, err)
return err
}

return nil
}

Expand All @@ -1292,15 +1290,15 @@ func (c *Controller) reconcileCustomVpcAddNormalStaticRoute(vpcName string) erro
return err
}
gatewayV4, gatewayV6 := util.SplitStringIP(defualtExternalSubnet.Spec.Gateway)
vpc, err := c.vpcsLister.Get(vpcName)
cachedVpc, err := c.vpcsLister.Get(vpcName)
if err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
klog.Errorf("failed to get vpc %s, %v", vpcName, err)
return err
}

vpc := cachedVpc.DeepCopy()
rtbs := c.getRouteTablesByVpc(vpc)
routeTotal := len(vpc.Spec.StaticRoutes) + len(rtbs)*2
routes := make([]*kubeovnv1.StaticRoute, 0, routeTotal)
Expand Down Expand Up @@ -1351,17 +1349,15 @@ func (c *Controller) reconcileCustomVpcAddNormalStaticRoute(vpcName string) erro

if needUpdate {
vpc.Spec.StaticRoutes = routes
if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}); err != nil {
if vpc, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}); err != nil {
klog.Errorf("failed to update vpc spec static route %s, %v", vpc.Name, err)
return err
}
if err = c.patchVpcBfdStatus(vpc.Name); err != nil {
klog.Errorf("failed to patch vpc %s, %v", vpc.Name, err)
return err
}
}

if err = c.patchVpcBfdStatus(vpc.Name); err != nil {
klog.Errorf("failed to patch vpc %s, %v", vpc.Name, err)
return err
}

return nil
}

Expand Down
Loading

0 comments on commit 6ba997d

Please sign in to comment.