Skip to content

Commit

Permalink
fix eip nat clean
Browse files Browse the repository at this point in the history
  • Loading branch information
@bobz965
bobz965 committed Aug 15, 2023
1 parent 3b5d53e commit fbbd75e
Show file tree
Hide file tree
Showing 5 changed files with 262 additions and 61 deletions.
95 changes: 80 additions & 15 deletions pkg/controller/ovn_dnat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/util"
Expand Down Expand Up @@ -63,8 +64,7 @@ func (c *Controller) enqueueDelOvnDnatRule(obj interface{}) {
return
}
klog.Infof("enqueue delete ovn dnat %s", key)
dnat := obj.(*kubeovnv1.OvnDnatRule)
c.delOvnDnatRuleQueue.Add(dnat)
c.delOvnDnatRuleQueue.Add(key)
}

func (c *Controller) runAddOvnDnatRuleWorker() {
Expand Down Expand Up @@ -150,16 +150,16 @@ func (c *Controller) processNextDeleteOvnDnatRuleWorkItem() bool {

err := func(obj interface{}) error {
defer c.delOvnDnatRuleQueue.Done(obj)
var dnat *kubeovnv1.OvnDnatRule
var key string
var ok bool
if dnat, ok = obj.(*kubeovnv1.OvnDnatRule); !ok {
if key, ok = obj.(string); !ok {
c.delOvnDnatRuleQueue.Forget(obj)
utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj))
return nil
}
if err := c.handleDelOvnDnatRule(dnat); err != nil {
c.delOvnDnatRuleQueue.AddRateLimited(obj)
return fmt.Errorf("error syncing '%s': %s, requeuing", dnat.Name, err.Error())
if err := c.handleDelOvnDnatRule(key); err != nil {
c.delOvnDnatRuleQueue.AddRateLimited(key)
return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
}
c.delOvnDnatRuleQueue.Forget(obj)
return nil
Expand Down Expand Up @@ -274,7 +274,7 @@ func (c *Controller) handleAddOvnDnatRule(key string) error {
return err
}

if err = c.handleAddOvnEipFinalizer(cachedEip, util.OvnEipFinalizer); err != nil {
if err = c.handleAddOvnEipFinalizer(cachedEip, util.ControllerName); err != nil {
klog.Errorf("failed to add finalizer for ovn eip, %v", err)
return err
}
Expand All @@ -285,6 +285,11 @@ func (c *Controller) handleAddOvnDnatRule(key string) error {
return err
}

if err := c.handleAddOvnDnatFinalizer(cachedDnat, util.ControllerName); err != nil {
klog.Errorf("failed to add finalizer for ovn dnat %s, %v", cachedDnat.Name, err)
return err
}

// patch dnat eip relationship
if err = c.natLabelAndAnnoOvnEip(eipName, cachedDnat.Name, vpcName); err != nil {
klog.Errorf("failed to label dnat '%s' in eip %s, %v", cachedDnat.Name, eipName, err)
Expand All @@ -310,16 +315,29 @@ func (c *Controller) handleAddOvnDnatRule(key string) error {
return nil
}

func (c *Controller) handleDelOvnDnatRule(dnat *kubeovnv1.OvnDnatRule) error {
if dnat.Status.Vpc != "" && dnat.Status.V4Eip != "" && dnat.Status.ExternalPort != "" {
if err := c.DelDnatRule(dnat.Status.Vpc, dnat.Name,
dnat.Status.V4Eip, dnat.Status.ExternalPort); err != nil {
klog.Errorf("failed to delete dnat, %v", err)
func (c *Controller) handleDelOvnDnatRule(key string) error {
cachedDnat, err := c.ovnDnatRulesLister.Get(key)
if err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
klog.Error(err)
return err
}
klog.Infof("handle delete ovn dnat %s", key)
if cachedDnat.Status.Vpc != "" && cachedDnat.Status.V4Eip != "" && cachedDnat.Status.ExternalPort != "" {
if err = c.DelDnatRule(cachedDnat.Status.Vpc, cachedDnat.Name,
cachedDnat.Status.V4Eip, cachedDnat.Status.ExternalPort); err != nil {
klog.Errorf("failed to delete dnat %s, %v", key, err)
return err
}
}
if dnat.Spec.OvnEip != "" {
c.resetOvnEipQueue.Add(dnat.Spec.OvnEip)
if err = c.handleDelOvnDnatFinalizer(cachedDnat, util.ControllerName); err != nil {
klog.Errorf("failed to remove finalizer for ovn dnat %s, %v", cachedDnat.Name, err)
return err
}
if cachedDnat.Spec.OvnEip != "" {
c.resetOvnEipQueue.Add(cachedDnat.Spec.OvnEip)
}
return nil
}
Expand Down Expand Up @@ -592,3 +610,50 @@ func (c *Controller) DelDnatRule(vpcName, dnatName, externalIp, externalPort str

return nil
}

func (c *Controller) handleAddOvnDnatFinalizer(cachedDnat *kubeovnv1.OvnDnatRule, finalizer string) error {
if cachedDnat.DeletionTimestamp.IsZero() {
if util.ContainsString(cachedDnat.Finalizers, finalizer) {
return nil
}
}
newDnat := cachedDnat.DeepCopy()
controllerutil.AddFinalizer(newDnat, finalizer)
patch, err := util.GenerateMergePatchPayload(cachedDnat, newDnat)
if err != nil {
klog.Errorf("failed to generate patch payload for ovn dnat '%s', %v", cachedDnat.Name, err)
return err
}
if _, err := c.config.KubeOvnClient.KubeovnV1().OvnDnatRules().Patch(context.Background(), cachedDnat.Name,
types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
klog.Errorf("failed to add finalizer for ovn dnat '%s', %v", cachedDnat.Name, err)
return err
}
return nil
}

func (c *Controller) handleDelOvnDnatFinalizer(cachedDnat *kubeovnv1.OvnDnatRule, finalizer string) error {
if len(cachedDnat.Finalizers) == 0 {
return nil
}
var err error
newDnat := cachedDnat.DeepCopy()
controllerutil.RemoveFinalizer(newDnat, finalizer)
patch, err := util.GenerateMergePatchPayload(cachedDnat, newDnat)
if err != nil {
klog.Errorf("failed to generate patch payload for ovn dnat '%s', %v", cachedDnat.Name, err)
return err
}
if _, err := c.config.KubeOvnClient.KubeovnV1().OvnDnatRules().Patch(context.Background(), cachedDnat.Name,
types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
klog.Errorf("failed to remove finalizer from ovn dnat '%s', %v", cachedDnat.Name, err)
return err
}
return nil
}
36 changes: 22 additions & 14 deletions pkg/controller/ovn_eip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,7 @@ func (c *Controller) enqueueDelOvnEip(obj interface{}) {
return
}
klog.Infof("enqueue del ovn eip %s", key)
eip := obj.(*kubeovnv1.OvnEip)
c.delOvnEipQueue.Add(eip)
c.delOvnEipQueue.Add(key)
}

func (c *Controller) runAddOvnEipWorker() {
Expand Down Expand Up @@ -189,16 +188,16 @@ func (c *Controller) processNextDeleteOvnEipWorkItem() bool {
}
err := func(obj interface{}) error {
defer c.delOvnEipQueue.Done(obj)
var eip *kubeovnv1.OvnEip
var key string
var ok bool
if eip, ok = obj.(*kubeovnv1.OvnEip); !ok {
if key, ok = obj.(string); !ok {
c.delOvnEipQueue.Forget(obj)
utilruntime.HandleError(fmt.Errorf("expected ovn eip in workqueue but got %#v", obj))
return nil
}
if err := c.handleDelOvnEip(eip); err != nil {
c.delOvnEipQueue.AddRateLimited(obj)
return fmt.Errorf("error syncing '%s': %s, requeuing", eip.Name, err.Error())
if err := c.handleDelOvnEip(key); err != nil {
c.delOvnEipQueue.AddRateLimited(key)
return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
}
c.delOvnEipQueue.Forget(obj)
return nil
Expand Down Expand Up @@ -285,7 +284,7 @@ func (c *Controller) handleUpdateOvnEip(key string) error {
klog.Error(err)
return err
}
klog.V(3).Infof("handle update ovn eip %s", cachedEip.Name)
klog.Infof("handle update ovn eip %s", cachedEip.Name)
if !cachedEip.DeletionTimestamp.IsZero() {
subnetName := cachedEip.Spec.ExternalSubnet
if subnetName == "" {
Expand Down Expand Up @@ -332,28 +331,37 @@ func (c *Controller) handleResetOvnEip(key string) error {
return nil
}

func (c *Controller) handleDelOvnEip(eip *kubeovnv1.OvnEip) error {
klog.V(3).Infof("handle del ovn eip %s", eip.Name)
func (c *Controller) handleDelOvnEip(key string) error {
klog.Infof("handle del ovn eip %s", key)
eip, err := c.ovnEipsLister.Get(key)
if err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
klog.Error(err)
return err
}

if len(eip.Finalizers) > 1 {
err := errors.New("eip is referenced, it cannot be deleted directly")
err = errors.New("eip is referenced, it cannot be deleted directly")
klog.Errorf("failed to delete eip %s, %v", eip.Name, err)
return err
}

if err := c.handleDelOvnEipFinalizer(eip, util.OvnEipFinalizer); err != nil {
if err = c.handleDelOvnEipFinalizer(eip, util.ControllerName); err != nil {
klog.Errorf("failed to handle remove ovn eip finalizer , %v", err)
return err
}

if eip.Spec.Type == util.Lsp {
if err := c.ovnClient.DeleteLogicalSwitchPort(eip.Name); err != nil {
if err = c.ovnClient.DeleteLogicalSwitchPort(eip.Name); err != nil {
klog.Errorf("failed to delete lsp %s, %v", eip.Name, err)
return err
}
}

if eip.Spec.Type == util.Lrp {
if err := c.ovnClient.DeleteLogicalRouterPort(eip.Name); err != nil {
if err = c.ovnClient.DeleteLogicalRouterPort(eip.Name); err != nil {
klog.Errorf("failed to delete lrp %s, %v", eip.Name, err)
return err
}
Expand Down
97 changes: 81 additions & 16 deletions pkg/controller/ovn_fip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
Expand Down Expand Up @@ -62,8 +63,7 @@ func (c *Controller) enqueueDelOvnFip(obj interface{}) {
return
}
klog.Infof("enqueue del ovn fip %s", key)
fip := obj.(*kubeovnv1.OvnFip)
c.delOvnFipQueue.Add(fip)
c.delOvnFipQueue.Add(key)
}

func (c *Controller) runAddOvnFipWorker() {
Expand Down Expand Up @@ -146,16 +146,16 @@ func (c *Controller) processNextDeleteOvnFipWorkItem() bool {
}
err := func(obj interface{}) error {
defer c.delOvnFipQueue.Done(obj)
var fip *kubeovnv1.OvnFip
var key string
var ok bool
if fip, ok = obj.(*kubeovnv1.OvnFip); !ok {
if key, ok = obj.(string); !ok {
c.delOvnFipQueue.Forget(obj)
utilruntime.HandleError(fmt.Errorf("expected fip in workqueue but got %#v", obj))
return nil
}
if err := c.handleDelOvnFip(fip); err != nil {
c.delOvnFipQueue.AddRateLimited(obj)
return fmt.Errorf("error syncing '%s': %s, requeuing", fip.Name, err.Error())
if err := c.handleDelOvnFip(key); err != nil {
c.delOvnFipQueue.AddRateLimited(key)
return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
}
c.delOvnFipQueue.Forget(obj)
return nil
Expand Down Expand Up @@ -272,7 +272,7 @@ func (c *Controller) handleAddOvnFip(key string) error {
klog.Errorf("failed to patch status for fip %s, %v", key, err)
return err
}
if err = c.handleAddOvnEipFinalizer(cachedEip, util.OvnEipFinalizer); err != nil {
if err = c.handleAddOvnEipFinalizer(cachedEip, util.ControllerName); err != nil {
klog.Errorf("failed to add finalizer for ovn eip, %v", err)
return err
}
Expand All @@ -283,6 +283,12 @@ func (c *Controller) handleAddOvnFip(key string) error {
klog.Errorf("failed to create v4 fip, %v", err)
return err
}

if err = c.handleAddOvnFipFinalizer(cachedFip, util.ControllerName); err != nil {
klog.Errorf("failed to add finalizer for ovn fip, %v", err)
return err
}

// patch fip eip relationship
if err = c.natLabelAndAnnoOvnEip(eipName, cachedFip.Name, vpcName); err != nil {
klog.Errorf("failed to label fip '%s' in eip %s, %v", cachedFip.Name, eipName, err)
Expand Down Expand Up @@ -406,18 +412,30 @@ func (c *Controller) handleUpdateOvnFip(key string) error {
return nil
}

func (c *Controller) handleDelOvnFip(fip *kubeovnv1.OvnFip) error {
klog.Infof("handle del ovn fip %s", fip.Name)
// ovn delete fip
if fip.Status.Vpc != "" && fip.Status.V4Eip != "" && fip.Status.V4Ip != "" {
if err := c.ovnClient.DeleteNat(fip.Status.Vpc, ovnnb.NATTypeDNATAndSNAT, fip.Status.V4Eip, fip.Status.V4Ip); err != nil {
klog.Errorf("failed to delete fip, %v", err)
func (c *Controller) handleDelOvnFip(key string) error {
klog.Infof("handle del ovn fip %s", key)
cachedFip, err := c.ovnFipsLister.Get(key)
if err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
klog.Error(err)
return err
}
// ovn delete fip nat
if cachedFip.Status.Vpc != "" && cachedFip.Status.V4Eip != "" && cachedFip.Status.V4Ip != "" {
if err = c.ovnClient.DeleteNat(cachedFip.Status.Vpc, ovnnb.NATTypeDNATAndSNAT, cachedFip.Status.V4Eip, cachedFip.Status.V4Ip); err != nil {
klog.Errorf("failed to delete fip %s, %v", key, err)
return err
}
}
if err = c.handleDelOvnFipFinalizer(cachedFip, util.ControllerName); err != nil {
klog.Errorf("failed to remove finalizer for ovn fip %s, %v", cachedFip.Name, err)
return err
}
// reset eip
if fip.Spec.OvnEip != "" {
c.resetOvnEipQueue.Add(fip.Spec.OvnEip)
if cachedFip.Spec.OvnEip != "" {
c.resetOvnEipQueue.Add(cachedFip.Spec.OvnEip)
}
return nil
}
Expand Down Expand Up @@ -544,3 +562,50 @@ func (c *Controller) GetOvnEip(eipName string) (*kubeovnv1.OvnEip, error) {
}
return cachedEip, nil
}

func (c *Controller) handleAddOvnFipFinalizer(cachedFip *kubeovnv1.OvnFip, finalizer string) error {
if cachedFip.DeletionTimestamp.IsZero() {
if util.ContainsString(cachedFip.Finalizers, finalizer) {
return nil
}
}
newFip := cachedFip.DeepCopy()
controllerutil.AddFinalizer(newFip, finalizer)
patch, err := util.GenerateMergePatchPayload(cachedFip, newFip)
if err != nil {
klog.Errorf("failed to generate patch payload for ovn fip '%s', %v", cachedFip.Name, err)
return err
}
if _, err := c.config.KubeOvnClient.KubeovnV1().OvnFips().Patch(context.Background(), cachedFip.Name,
types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
klog.Errorf("failed to add finalizer for ovn fip '%s', %v", cachedFip.Name, err)
return err
}
return nil
}

func (c *Controller) handleDelOvnFipFinalizer(cachedFip *kubeovnv1.OvnFip, finalizer string) error {
if len(cachedFip.Finalizers) == 0 {
return nil
}
var err error
newFip := cachedFip.DeepCopy()
controllerutil.RemoveFinalizer(newFip, finalizer)
patch, err := util.GenerateMergePatchPayload(cachedFip, newFip)
if err != nil {
klog.Errorf("failed to generate patch payload for ovn fip '%s', %v", cachedFip.Name, err)
return err
}
if _, err := c.config.KubeOvnClient.KubeovnV1().OvnFips().Patch(context.Background(), cachedFip.Name,
types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
klog.Errorf("failed to remove finalizer from ovn fip '%s', %v", cachedFip.Name, err)
return err
}
return nil
}
Loading

0 comments on commit fbbd75e

Please sign in to comment.