From 5d5837d8d54482eaa7be8c9b40b74c9521361f6c Mon Sep 17 00:00:00 2001 From: yanjianbo Date: Sat, 7 Oct 2023 14:34:36 +0800 Subject: [PATCH 1/2] Improve the way to maintain iptables rules Signed-off-by: yanjianbo --- pkg/cloud-agent/cloud_agent.go | 9 +- pkg/connector/iptables.go | 227 +++++++++------------------------ pkg/connector/manager.go | 28 ++-- pkg/util/ipset/ipset.go | 34 +++++ pkg/util/iptables/iptables.go | 208 ++++++++++++++++++++++++++++++ 5 files changed, 321 insertions(+), 185 deletions(-) create mode 100644 pkg/util/iptables/iptables.go diff --git a/pkg/cloud-agent/cloud_agent.go b/pkg/cloud-agent/cloud_agent.go index da64153..31d46af 100644 --- a/pkg/cloud-agent/cloud_agent.go +++ b/pkg/cloud-agent/cloud_agent.go @@ -149,12 +149,13 @@ func (a *CloudAgent) addAndSaveRoutes(cp routing.ConnectorPrefixes) { } routes := a.syncRoutes(cp.LocalPrefixes, cp.RemotePrefixes) - routes = a.syncRoutes(cp.LocalPrefixes, cp.EdgeNodeCIDRs) + routes = append(routes, a.syncRoutes(cp.LocalPrefixes, cp.EdgeNodeCIDRs)...) routes = append(routes, a.syncRoutes(cp.LocalPrefixes6, cp.RemotePrefixes6)...) - whitelist := sets.NewString(cp.RemotePrefixes...) - whitelist.Insert(cp.EdgeNodeCIDRs...) - whitelist.Insert(cp.RemotePrefixes6...) + whitelist := sets.NewString() + for _, route := range routes { + whitelist.Insert(route.Dst.String()) + } if err := routeutil.PurgeStrongSwanRoutes(routeutil.NewDstWhitelist(whitelist)); err != nil { logger.Error(err, "failed to purge stale routes in strongswan table") } diff --git a/pkg/connector/iptables.go b/pkg/connector/iptables.go index 85fa8a5..a8c5667 100644 --- a/pkg/connector/iptables.go +++ b/pkg/connector/iptables.go @@ -15,35 +15,51 @@ package connector import ( + "bytes" "sync" + "text/template" - "github.com/coreos/go-iptables/iptables" "github.com/fabedge/fabedge/pkg/util/ipset" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2/klogr" -) - -const ( - TableFilter = "filter" - TableNat = "nat" - ChainInput = "INPUT" - ChainForward = "FORWARD" - ChainPostRouting = "POSTROUTING" - ChainFabEdgeInput = "FABEDGE-INPUT" - ChainFabEdgeForward = "FABEDGE-FORWARD" - ChainFabEdgePostRouting = "FABEDGE-POSTROUTING" - IPSetEdgePodCIDR = "FABEDGE-EDGE-POD-CIDR" - IPSetEdgePodCIDR6 = "FABEDGE-EDGE-POD-CIDR6" - IPSetEdgeNodeCIDR = "FABEDGE-EDGE-NODE-CIDR" - IPSetEdgeNodeCIDR6 = "FABEDGE-EDGE-NODE-CIDR6" - IPSetCloudPodCIDR = "FABEDGE-CLOUD-POD-CIDR" - IPSetCloudPodCIDR6 = "FABEDGE-CLOUD-POD-CIDR6" - IPSetCloudNodeCIDR = "FABEDGE-CLOUD-NODE-CIDR" - IPSetCloudNodeCIDR6 = "FABEDGE-CLOUD-NODE-CIDR6" + "github.com/fabedge/fabedge/pkg/util/iptables" ) +var tmpl = template.Must(template.New("iptables").Parse(` +*filter +:FABEDGE-INPUT - [0:0] +:FABEDGE-FORWARD - [0:0] + +-A FABEDGE-INPUT -p udp -m udp --dport 500 -j ACCEPT +-A FABEDGE-INPUT -p udp -m udp --dport 4500 -j ACCEPT +-A FABEDGE-INPUT -p esp -j ACCEPT +-A FABEDGE-INPUT -p ah -j ACCEPT + +-A FABEDGE-FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A FABEDGE-FORWARD -m set --match-set {{ .CloudPodCIDR }} src -j ACCEPT +-A FABEDGE-FORWARD -m set --match-set {{ .CloudPodCIDR }} dst -j ACCEPT +-A FABEDGE-FORWARD -m set --match-set {{ .CloudNodeCIDR }} src -j ACCEPT +-A FABEDGE-FORWARD -m set --match-set {{ .CloudNodeCIDR }} dst -j ACCEPT +COMMIT + +*nat +:FABEDGE-POSTROUTING - [0:0] +-A FABEDGE-POSTROUTING -m set --match-set {{ .CloudPodCIDR }} src -m set --match-set {{ .EdgePodCIDR}} dst -j ACCEPT +-A FABEDGE-POSTROUTING -m set --match-set {{ .EdgePodCIDR }} src -m set --match-set {{ .CloudPodCIDR }} dst -j ACCEPT +-A FABEDGE-POSTROUTING -m set --match-set {{ .CloudPodCIDR }} src -m set --match-set {{ .EdgeNodeCIDR }} dst -j ACCEPT +-A FABEDGE-POSTROUTING -m set --match-set {{ .EdgePodCIDR }} src -m set --match-set {{ .CloudNodeCIDR }} dst -j MASQUERADE +-A FABEDGE-POSTROUTING -m set --match-set {{ .EdgeNodeCIDR }} src -m set --match-set {{ .CloudPodCIDR}} dst -j MASQUERADE +COMMIT +`)) + +var jumpChains = []iptables.JumpChain{ + {Table: iptables.TableFilter, SrcChain: iptables.ChainInput, DstChain: iptables.ChainFabEdgeInput, Position: iptables.Append}, + {Table: iptables.TableFilter, SrcChain: iptables.ChainForward, DstChain: iptables.ChainFabEdgeForward, Position: iptables.Append}, + {Table: iptables.TableNAT, SrcChain: iptables.ChainPostRouting, DstChain: iptables.ChainFabEdgePostRouting, Position: iptables.Prepend}, +} + type IPSetSpec struct { Name string EntrySet sets.String @@ -57,54 +73,49 @@ type IPSetNames struct { } type IPTablesHandler struct { - ipt *iptables.IPTables + ipt iptables.ApplierCleaner ipset ipset.Interface log logr.Logger - names IPSetNames + names ipset.IPSetNames hashFamily string + rulesData []byte + specs []IPSetSpec lock sync.RWMutex } func newIP4TablesHandler() (*IPTablesHandler, error) { - ipt, err := iptables.New() - if err != nil { + names := ipset.Names4 + rulesData := bytes.NewBuffer(nil) + if err := tmpl.Execute(rulesData, names); err != nil { return nil, err } return &IPTablesHandler{ - log: klogr.New().WithName("iptablesHandler"), - ipt: ipt, + log: klogr.New().WithName("iptables-handler"), + ipt: iptables.NewApplierCleaner(iptables.ProtocolIPv4, jumpChains, rulesData.Bytes()), ipset: ipset.New(), hashFamily: ipset.ProtocolFamilyIPV4, - names: IPSetNames{ - EdgeNodeCIDR: IPSetEdgeNodeCIDR, - EdgePodCIDR: IPSetEdgePodCIDR, - CloudPodCIDR: IPSetCloudPodCIDR, - CloudNodeCIDR: IPSetCloudNodeCIDR, - }, + names: names, + rulesData: rulesData.Bytes(), }, nil } func newIP6TablesHandler() (*IPTablesHandler, error) { - ipt, err := iptables.New(iptables.IPFamily(iptables.ProtocolIPv6)) - if err != nil { + names := ipset.Names6 + rulesData := bytes.NewBuffer(nil) + if err := tmpl.Execute(rulesData, names); err != nil { return nil, err } return &IPTablesHandler{ - log: klogr.New().WithName("ip6tablesHandler"), - ipt: ipt, + log: klogr.New().WithName("ip6tables-handler"), + ipt: iptables.NewApplierCleaner(iptables.ProtocolIPv6, jumpChains, rulesData.Bytes()), ipset: ipset.New(), hashFamily: ipset.ProtocolFamilyIPV6, - names: IPSetNames{ - EdgeNodeCIDR: IPSetEdgeNodeCIDR6, - EdgePodCIDR: IPSetEdgePodCIDR6, - CloudPodCIDR: IPSetCloudPodCIDR6, - CloudNodeCIDR: IPSetCloudNodeCIDR6, - }, + names: names, }, nil } @@ -132,128 +143,6 @@ func (h *IPTablesHandler) setIPSetEntrySet(edgePodCIDRSet, edgeNodeCIDRSet, clou } } -func (h *IPTablesHandler) CleanSNatIPTablesRules() error { - return h.ipt.ClearChain(TableNat, ChainFabEdgePostRouting) -} - -func (h *IPTablesHandler) clearFabEdgeIptablesChains() error { - err := h.ipt.ClearChain(TableFilter, ChainFabEdgeInput) - if err != nil { - return err - } - err = h.ipt.ClearChain(TableFilter, ChainFabEdgeForward) - if err != nil { - return err - } - return h.ipt.ClearChain(TableNat, ChainFabEdgePostRouting) -} - -func (h *IPTablesHandler) ensureForwardIPTablesRules() (err error) { - // ensure rules exist - if err = h.ipt.AppendUnique(TableFilter, ChainForward, "-j", ChainFabEdgeForward); err != nil { - return err - } - - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeForward, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil { - return err - } - - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeForward, "-m", "set", "--match-set", h.names.CloudPodCIDR, "src", "-j", "ACCEPT"); err != nil { - return err - } - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeForward, "-m", "set", "--match-set", h.names.CloudPodCIDR, "dst", "-j", "ACCEPT"); err != nil { - return err - } - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeForward, "-m", "set", "--match-set", h.names.CloudNodeCIDR, "src", "-j", "ACCEPT"); err != nil { - return err - } - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeForward, "-m", "set", "--match-set", h.names.CloudNodeCIDR, "dst", "-j", "ACCEPT"); err != nil { - return err - } - - return nil -} - -func (h *IPTablesHandler) ensureNatIPTablesRules() (err error) { - if err = h.ipt.ClearChain(TableNat, ChainFabEdgePostRouting); err != nil { - return err - } - exists, err := h.ipt.Exists(TableNat, ChainPostRouting, "-j", ChainFabEdgePostRouting) - if err != nil { - return err - } - - if !exists { - if err = h.ipt.Insert(TableNat, ChainPostRouting, 1, "-j", ChainFabEdgePostRouting); err != nil { - return err - } - } - - // for cloud-pod to edge-pod, not masquerade, in order to avoid flannel issue - if err = h.ipt.AppendUnique(TableNat, ChainFabEdgePostRouting, "-m", "set", "--match-set", h.names.CloudPodCIDR, "src", "-m", "set", "--match-set", h.names.EdgePodCIDR, "dst", "-j", "ACCEPT"); err != nil { - return err - } - - // for edge-pod to cloud-pod, not masquerade, in order to avoid flannel issue - if err = h.ipt.AppendUnique(TableNat, ChainFabEdgePostRouting, "-m", "set", "--match-set", h.names.EdgePodCIDR, "src", "-m", "set", "--match-set", h.names.CloudPodCIDR, "dst", "-j", "ACCEPT"); err != nil { - return err - } - - // for cloud-pod to edge-node, not masquerade, in order to avoid flannel issue - if err = h.ipt.AppendUnique(TableNat, ChainFabEdgePostRouting, "-m", "set", "--match-set", h.names.CloudPodCIDR, "src", "-m", "set", "--match-set", h.names.EdgeNodeCIDR, "dst", "-j", "ACCEPT"); err != nil { - return err - } - - // for edge-pod to cloud-node, to masquerade it, in order to avoid rp_filter issue - if err = h.ipt.AppendUnique(TableNat, ChainFabEdgePostRouting, "-m", "set", "--match-set", h.names.EdgePodCIDR, "src", "-m", "set", "--match-set", h.names.CloudNodeCIDR, "dst", "-j", "MASQUERADE"); err != nil { - return err - } - - // for edge-node to cloud-pod, to masquerade it, or the return traffic will not come back to connector node. - return h.ipt.AppendUnique(TableNat, ChainFabEdgePostRouting, "-m", "set", "--match-set", h.names.EdgeNodeCIDR, "src", "-m", "set", "--match-set", h.names.CloudPodCIDR, "dst", "-j", "MASQUERADE") - -} - -func (h *IPTablesHandler) ensureIPSpecInputRules() (err error) { - if err = h.ipt.AppendUnique(TableFilter, ChainInput, "-j", ChainFabEdgeInput); err != nil { - return err - } - - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeInput, "-p", "udp", "-m", "udp", "--dport", "500", "-j", "ACCEPT"); err != nil { - return err - } - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeInput, "-p", "udp", "-m", "udp", "--dport", "4500", "-j", "ACCEPT"); err != nil { - return err - } - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeInput, "-p", "esp", "-j", "ACCEPT"); err != nil { - return err - } - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeInput, "-p", "ah", "-j", "ACCEPT"); err != nil { - return err - } - return nil -} - -func (h *IPTablesHandler) maintainIPTables() { - if err := h.ensureForwardIPTablesRules(); err != nil { - h.log.Error(err, "failed to add iptables forward rules") - } else { - h.log.V(5).Info("iptables forward rules are added") - } - - if err := h.ensureNatIPTablesRules(); err != nil { - h.log.Error(err, "failed to add iptables nat rules") - } else { - h.log.V(5).Info("iptables nat rules are added") - } - - if err := h.ensureIPSpecInputRules(); err != nil { - h.log.Error(err, "failed to add iptables input rules") - } else { - h.log.V(5).Info("iptables input rules are added") - } -} - func (h *IPTablesHandler) maintainIPSet() { var specs []IPSetSpec @@ -276,13 +165,21 @@ func (h *IPTablesHandler) maintainIPSet() { } } +func (h *IPTablesHandler) maintainIPTables() { + h.maintainIPSet() + + if err := h.ipt.Apply(); err != nil { + h.log.Error(err, "failed to restore iptables rules") + } +} + func (h *IPTablesHandler) getEdgeNodeCIDRs() []string { h.lock.RLock() specs := h.specs h.lock.RUnlock() for _, spec := range specs { - if spec.Name == IPSetEdgeNodeCIDR { + if spec.Name == ipset.IPSetEdgeNodeCIDR { return spec.EntrySet.List() } } diff --git a/pkg/connector/manager.go b/pkg/connector/manager.go index 4086ab5..3e21020 100644 --- a/pkg/connector/manager.go +++ b/pkg/connector/manager.go @@ -180,7 +180,7 @@ func (m *Manager) notify() { func (m *Manager) Start() { about.DisplayVersion() - m.clearFabEdgeIptablesChains() + m.removeAllChains() go m.runLeaderElection() go m.runHTTPServer() @@ -239,34 +239,33 @@ func (m *Manager) runLeaderElection() { func (m *Manager) gracefulShutdown() { err := m.router.CleanRoutes(m.connections) if err != nil { - m.log.Error(err, "failed to clean routers") + m.log.Error(err, "failed to clean routes") } - m.cleanSNatIPTablesRules() + m.removeAllChains() } func (m *Manager) clearAll() { err := m.router.CleanRoutes(m.connections) if err != nil { - m.log.Error(err, "failed to clean routers") + m.log.Error(err, "failed to clean routes") } - m.cleanSNatIPTablesRules() - m.clearFabEdgeIptablesChains() + m.flushAllChains() m.clearConnections() } -func (m *Manager) cleanSNatIPTablesRules() { - for _, ipt := range []*IPTablesHandler{m.iptHandler, m.ipt6Handler} { - if err := ipt.CleanSNatIPTablesRules(); err != nil { - m.log.Error(err, "failed to clean iptables") +func (m *Manager) flushAllChains() { + for _, h := range []*IPTablesHandler{m.iptHandler, m.ipt6Handler} { + if err := h.ipt.Flush(); err != nil { + m.log.Error(err, "failed to flush iptables rules") } } } -func (m *Manager) clearFabEdgeIptablesChains() { - for _, ipt := range []*IPTablesHandler{m.iptHandler, m.ipt6Handler} { - if err := ipt.clearFabEdgeIptablesChains(); err != nil { +func (m *Manager) removeAllChains() { + for _, h := range []*IPTablesHandler{m.iptHandler, m.ipt6Handler} { + if err := h.ipt.Remove(); err != nil { m.log.Error(err, "failed to clean iptables") } } @@ -333,10 +332,7 @@ func (m *Manager) workLoop() { m.maintainTunnels() m.maintainRoutes() - m.iptHandler.maintainIPSet() m.iptHandler.maintainIPTables() - - m.ipt6Handler.maintainIPSet() m.ipt6Handler.maintainIPTables() m.broadcastConnectorPrefixes() diff --git a/pkg/util/ipset/ipset.go b/pkg/util/ipset/ipset.go index 1149d41..d38c8ea 100644 --- a/pkg/util/ipset/ipset.go +++ b/pkg/util/ipset/ipset.go @@ -36,6 +36,40 @@ const ( ProtocolFamilyIPV6 = ipset.ProtocolFamilyIPV6 ) +const ( + IPSetEdgePodCIDR = "FABEDGE-EDGE-POD-CIDR" + IPSetEdgePodCIDR6 = "FABEDGE-EDGE-POD-CIDR6" + IPSetEdgeNodeCIDR = "FABEDGE-EDGE-NODE-CIDR" + IPSetEdgeNodeCIDR6 = "FABEDGE-EDGE-NODE-CIDR6" + IPSetCloudPodCIDR = "FABEDGE-CLOUD-POD-CIDR" + IPSetCloudPodCIDR6 = "FABEDGE-CLOUD-POD-CIDR6" + IPSetCloudNodeCIDR = "FABEDGE-CLOUD-NODE-CIDR" + IPSetCloudNodeCIDR6 = "FABEDGE-CLOUD-NODE-CIDR6" +) + +type IPSetNames struct { + EdgePodCIDR string + EdgeNodeCIDR string + CloudPodCIDR string + CloudNodeCIDR string +} + +var ( + Names4 = IPSetNames{ + EdgeNodeCIDR: IPSetEdgeNodeCIDR, + EdgePodCIDR: IPSetEdgePodCIDR, + CloudPodCIDR: IPSetCloudPodCIDR, + CloudNodeCIDR: IPSetCloudNodeCIDR, + } + + Names6 = IPSetNames{ + EdgeNodeCIDR: IPSetEdgeNodeCIDR6, + EdgePodCIDR: IPSetEdgePodCIDR6, + CloudPodCIDR: IPSetCloudPodCIDR6, + CloudNodeCIDR: IPSetCloudNodeCIDR6, + } +) + type Interface interface { // EnsureIPSet ensure specified are created and ensure this ipset only contains all entries // specified in entrySet. diff --git a/pkg/util/iptables/iptables.go b/pkg/util/iptables/iptables.go new file mode 100644 index 0000000..d6ed5e7 --- /dev/null +++ b/pkg/util/iptables/iptables.go @@ -0,0 +1,208 @@ +// Copyright 2021 FabEdge Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iptables + +import ( + utilerrors "k8s.io/apimachinery/pkg/util/errors" + utiliptables "k8s.io/kubernetes/pkg/util/iptables" + utilexec "k8s.io/utils/exec" +) + +type Table = utiliptables.Table + +// iptables tables +const ( + TableFilter utiliptables.Table = "filter" + TableNAT = "nat" +) + +type Chain = utiliptables.Chain + +// iptables chains +const ( + ChainInput utiliptables.Chain = "INPUT" + ChainForward = "FORWARD" + ChainPostRouting = "POSTROUTING" + + ChainFabEdgeInput = "FABEDGE-INPUT" + ChainFabEdgeForward = "FABEDGE-FORWARD" + ChainFabEdgePostRouting = "FABEDGE-POSTROUTING" + ChainFabEdgeNatOutgoing = "FABEDGE-NAT-OUTGOING" +) + +type RulePosition = utiliptables.RulePosition + +const ( + Append RulePosition = utiliptables.Append + Prepend = utiliptables.Prepend +) + +type Protocol = utiliptables.Protocol + +const ( + ProtocolIPv4 Protocol = "IPv4" + ProtocolIPv6 Protocol = "IPv6" +) + +type FlushFlag = utiliptables.FlushFlag + +const ( + // FlushTables a boolean true constant for option flag FlushFlag + FlushTables FlushFlag = true + + // NoFlushTables a boolean false constant for option flag FlushFlag + NoFlushTables FlushFlag = false +) + +// RestoreCountersFlag is an option flag for Restore +type RestoreCountersFlag = utiliptables.RestoreCountersFlag + +const ( + RestoreCounters RestoreCountersFlag = true + NoRestoreCounters RestoreCountersFlag = false +) + +type JumpChain struct { + Table Table + SrcChain Chain + DstChain Chain + Position RulePosition +} + +type Interface interface { + utiliptables.Interface + // CreateChains create custom chains and insert them in specified positions + CreateChains(chains []JumpChain) error + // FlushAllChains flush rules of all custom chains + FlushAllChains(chains []JumpChain) error + // RemoveAllChains flush rules of all custom chains and remove them all + RemoveAllChains(chains []JumpChain) error + // RemoveChain flush rules of specified chain from specified table and remove the chain + RemoveChain(table Table, chain Chain) error + // NewApplierCleaner create a ApplierCleaner with specified custom chains and rules + NewApplierCleaner(chains []JumpChain, rulesData []byte) ApplierCleaner +} + +// ApplierCleaner is used to apply or clean custom chains and rules +type ApplierCleaner interface { + // Apply will create custom chains and insert rules + Apply() error + // Remove flush rules of all custom chains and remove them all + Remove() error + // Flush rules of all custom chains + Flush() error +} + +type iptablesHelper struct { + utiliptables.Interface + ipt utiliptables.Interface +} + +type applierCleaner struct { + helper Interface + chains []JumpChain + rulesData []byte +} + +func NewIPTablesHelper(protocol Protocol) Interface { + execer := utilexec.New() + ipt := utiliptables.New(execer, protocol) + return &iptablesHelper{ + ipt: utiliptables.New(execer, protocol), + Interface: ipt, + } +} + +func NewApplierCleaner(protocol Protocol, chains []JumpChain, rulesData []byte) ApplierCleaner { + return &applierCleaner{ + helper: NewIPTablesHelper(protocol), + chains: chains, + rulesData: rulesData, + } +} + +func (h *iptablesHelper) CreateChains(chains []JumpChain) error { + var errors []error + + for _, chain := range chains { + if _, err := h.ipt.EnsureChain(chain.Table, chain.DstChain); err != nil { + errors = append(errors, err) + continue + } + + _, err := h.ipt.EnsureRule(chain.Position, chain.Table, chain.SrcChain, "-j", string(chain.DstChain)) + if err != nil { + errors = append(errors, err) + } + } + + return utilerrors.NewAggregate(errors) +} + +func (h *iptablesHelper) RemoveAllChains(chains []JumpChain) error { + var errors []error + + for _, chain := range chains { + if err := h.RemoveChain(chain.Table, chain.DstChain); err != nil { + errors = append(errors, err) + } + } + + return utilerrors.NewAggregate(errors) +} + +func (h *iptablesHelper) FlushAllChains(chains []JumpChain) error { + var errors []error + + for _, chain := range chains { + if err := h.ipt.FlushChain(chain.Table, chain.DstChain); err != nil { + errors = append(errors, err) + } + } + + return utilerrors.NewAggregate(errors) +} + +func (h *iptablesHelper) RemoveChain(table Table, chain Chain) error { + if err := h.FlushChain(table, chain); err != nil { + return err + } + + return h.ipt.DeleteChain(table, chain) +} + +func (h *iptablesHelper) NewApplierCleaner(chains []JumpChain, rulesData []byte) ApplierCleaner { + return &applierCleaner{ + helper: h, + chains: chains, + rulesData: rulesData, + } +} + +func (ac *applierCleaner) Apply() error { + if err := ac.helper.CreateChains(ac.chains); err != nil { + return err + } + + return ac.helper.RestoreAll(ac.rulesData, NoFlushTables, RestoreCounters) +} + +func (ac *applierCleaner) Flush() error { + return ac.helper.FlushAllChains(ac.chains) +} + +func (ac *applierCleaner) Remove() error { + return ac.helper.RemoveAllChains(ac.chains) +} From 15846eb9ce02409dac1b5c2ba021f69790eb3d3a Mon Sep 17 00:00:00 2001 From: Zhen Tang Date: Fri, 13 Oct 2023 19:01:06 +0800 Subject: [PATCH 2/2] Use new iptables-helper in agent and clout-agent. Signed-off-by: Zhen Tang --- pkg/agent/config.go | 15 +-- pkg/agent/iptables.go | 166 +++++++++++++-------------------- pkg/agent/manager.go | 18 +--- pkg/cloud-agent/cloud_agent.go | 35 ++----- pkg/cloud-agent/iptables.go | 158 +++++++++++-------------------- pkg/connector/iptables.go | 2 +- pkg/connector/manager.go | 2 +- pkg/util/ipset/ipset.go | 5 + pkg/util/iptables/iptables.go | 38 ++++---- 9 files changed, 157 insertions(+), 282 deletions(-) diff --git a/pkg/agent/config.go b/pkg/agent/config.go index de80dd7..ef1549f 100644 --- a/pkg/agent/config.go +++ b/pkg/agent/config.go @@ -1,4 +1,4 @@ -// Copyright 2021 BoCloud +// Copyright 2021 FabEdge Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ import ( "time" debpkg "github.com/bep/debounce" - "github.com/coreos/go-iptables/iptables" "github.com/spf13/pflag" "k8s.io/klog/v2/klogr" "k8s.io/utils/exec" @@ -174,21 +173,9 @@ func (cfg Config) Manager() (*Manager, error) { return nil, err } - ipt, err := iptables.New() - if err != nil { - return nil, err - } - - ipt6, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) - if err != nil { - return nil, err - } - m := &Manager{ Config: cfg, tm: tm, - ipt4: ipt, - ipt6: ipt6, log: klogr.New().WithName("manager"), events: make(chan struct{}), diff --git a/pkg/agent/iptables.go b/pkg/agent/iptables.go index b25915f..9aca190 100644 --- a/pkg/agent/iptables.go +++ b/pkg/agent/iptables.go @@ -15,13 +15,12 @@ package agent import ( - "fmt" "strings" - "github.com/coreos/go-iptables/iptables" "k8s.io/apimachinery/pkg/util/sets" "github.com/fabedge/fabedge/pkg/util/ipset" + "github.com/fabedge/fabedge/pkg/util/iptables" ) type IPSet struct { @@ -29,6 +28,59 @@ type IPSet struct { EntrySet sets.String } +var jumpChains = []iptables.JumpChain{ + {Table: iptables.TableFilter, SrcChain: iptables.ChainForward, DstChain: iptables.ChainFabEdgeForward, Position: iptables.Append}, + {Table: iptables.TableNat, SrcChain: iptables.ChainPostRouting, DstChain: iptables.ChainFabEdgeNatOutgoing, Position: iptables.Prepend}, +} + +func buildRuleData(ipsetName string, subnets []string) []byte { + var builder strings.Builder + builder.WriteString(` +*filter +:FABEDGE-FORWARD - [0:0] + +`) + + for _, subnet := range subnets { + builder.WriteString("-A FABEDGE-FORWARD -s ") + builder.WriteString(subnet) + builder.WriteString(" -j ACCEPT\n") + + builder.WriteString("-A FABEDGE-FORWARD -d ") + builder.WriteString(subnet) + builder.WriteString(" -j ACCEPT\n") + } + + builder.WriteString(`COMMIT + +*nat +:FABEDGE-NAT-OUTGOING - [0:0] + +`) + + for _, subnet := range subnets { + builder.WriteString("-A FABEDGE-NAT-OUTGOING -s ") + builder.WriteString(subnet) + builder.WriteString(" -m set --match-set ") + builder.WriteString(ipsetName) + builder.WriteString(" dst -j RETURN\n") + + builder.WriteString("-A FABEDGE-NAT-OUTGOING -s ") + builder.WriteString(subnet) + builder.WriteString(" -d ") + builder.WriteString(subnet) + builder.WriteString(" -j RETURN\n") + + builder.WriteString("-A FABEDGE-NAT-OUTGOING -s ") + builder.WriteString(subnet) + builder.WriteString(" -j MASQUERADE\n") + } + + builder.WriteString("COMMIT\n") + + return []byte(builder.String()) +} + func (m *Manager) ensureIPTablesRules() error { current := m.getCurrentEndpoint() @@ -36,13 +88,12 @@ func (m *Manager) ensureIPTablesRules() error { subnetsIP4, subnetsIP6 := classifySubnets(current.Subnets) configs := []struct { - ipt *iptables.IPTables peerIPSet IPSet loopbackIPSet IPSet subnets []string + helper iptables.ApplierCleaner }{ { - ipt: m.ipt4, peerIPSet: IPSet{ IPSet: &ipset.IPSet{ Name: IPSetFabEdgePeerCIDR, @@ -52,9 +103,9 @@ func (m *Manager) ensureIPTablesRules() error { EntrySet: peerIPSet4, }, subnets: subnetsIP4, + helper: iptables.NewApplierCleaner(iptables.ProtocolIPv4, jumpChains, buildRuleData(IPSetFabEdgePeerCIDR, subnetsIP4)), }, { - ipt: m.ipt6, peerIPSet: IPSet{ IPSet: &ipset.IPSet{ Name: IPSetFabEdgePeerCIDR6, @@ -64,98 +115,20 @@ func (m *Manager) ensureIPTablesRules() error { EntrySet: peerIPSet6, }, subnets: subnetsIP6, + helper: iptables.NewApplierCleaner(iptables.ProtocolIPv6, jumpChains, buildRuleData(IPSetFabEdgePeerCIDR6, subnetsIP6)), }, } - clearOutgoingChain := !m.areSubnetsEqual(current.Subnets, m.lastSubnets) for _, c := range configs { - if err := m.ensureIPForwardRules(c.ipt, c.subnets); err != nil { - return err - } - - if m.MASQOutgoing { - if err := m.configureOutboundRules(c.ipt, c.peerIPSet, c.subnets, clearOutgoingChain); err != nil { - return err - } - } - } - // must be done after configureOutboundRules are executed - m.lastSubnets = current.Subnets - - return nil -} - -func (m *Manager) ensureIPForwardRules(ipt *iptables.IPTables, subnets []string) error { - if err := ensureChain(ipt, TableFilter, ChainFabEdgeForward); err != nil { - m.log.Error(err, "failed to check or create iptables chain", "table", TableFilter, "chain", ChainFabEdgeForward) - return err - } - - ensureRule := ipt.AppendUnique - if err := ensureRule(TableFilter, ChainForward, "-j", ChainFabEdgeForward); err != nil { - m.log.Error(err, "failed to check or add rule", "table", TableFilter, "chain", ChainForward, "rule", "-j FABEDGE") - return err - } - - // subnets won't change most of the time, and is append-only, so for now we don't need - // to handle removing old subnet - for _, subnet := range subnets { - if err := ensureRule(TableFilter, ChainFabEdgeForward, "-s", subnet, "-j", "ACCEPT"); err != nil { - m.log.Error(err, "failed to check or add rule", "table", TableFilter, "chain", ChainFabEdgeForward, "rule", fmt.Sprintf("-s %s -j ACCEPT", subnet)) - return err - } - - if err := ensureRule(TableFilter, ChainFabEdgeForward, "-d", subnet, "-j", "ACCEPT"); err != nil { - m.log.Error(err, "failed to check or add rule", "table", TableFilter, "chain", ChainFabEdgeForward, "rule", fmt.Sprintf("-d %s -j ACCEPT", subnet)) - return err - } - } - - return nil -} - -// outbound NAT from pods to outside the cluster -func (m *Manager) configureOutboundRules(ipt *iptables.IPTables, peerIPSet IPSet, subnets []string, clearFabEdgeNatOutgoingChain bool) error { - if clearFabEdgeNatOutgoingChain { - m.log.V(3).Info("Subnets are changed, clear iptables chain FABEDGE-NAT-OUTGOING") - if err := ipt.ClearChain(TableNat, ChainFabEdgeNatOutgoing); err != nil { - m.log.Error(err, "failed to check or add rule", "table", TableNat, "chain", ChainFabEdgeNatOutgoing) - return err - } - } else { - if err := ensureChain(ipt, TableNat, ChainFabEdgeNatOutgoing); err != nil { - m.log.Error(err, "failed to check or add rule", "table", TableNat, "chain", ChainFabEdgeNatOutgoing) + if err := m.ipset.EnsureIPSet(c.peerIPSet.IPSet, c.peerIPSet.EntrySet); err != nil { + m.log.Error(err, "failed to sync ipset", "ipsetName", c.peerIPSet.IPSet.Name) return err } - } - - if err := m.ipset.EnsureIPSet(peerIPSet.IPSet, peerIPSet.EntrySet); err != nil { - m.log.Error(err, "failed to sync ipset", "ipsetName", peerIPSet.IPSet.Name) - return err - } - for _, subnet := range subnets { - m.log.V(3).Info("configure outgoing NAT iptables rules") - - ensureRule := ipt.AppendUnique - if err := ensureRule(TableNat, ChainFabEdgeNatOutgoing, "-s", subnet, "-m", "set", "--match-set", peerIPSet.IPSet.Name, "dst", "-j", "RETURN"); err != nil { - m.log.Error(err, "failed to append rule", "table", TableNat, "chain", ChainFabEdgeNatOutgoing, "rule", fmt.Sprintf("-s %s -m set --match-set %s dst -j RETURN", subnet, peerIPSet.IPSet.Name)) - continue - } - - if err := ensureRule(TableNat, ChainFabEdgeNatOutgoing, "-s", subnet, "-d", subnet, "-j", "RETURN"); err != nil { - m.log.Error(err, "failed to append rule", "table", TableNat, "chain", ChainFabEdgeNatOutgoing, "rule", fmt.Sprintf("-s %s -d %s -j RETURN", subnet, subnet)) - continue - } - - if err := ensureRule(TableNat, ChainFabEdgeNatOutgoing, "-s", subnet, "-j", ChainMasquerade); err != nil { - m.log.Error(err, "failed to append rule", "table", TableNat, "chain", ChainFabEdgeNatOutgoing, "rule", fmt.Sprintf("-s %s -j %s", subnet, ChainMasquerade)) - continue - } - - if err := ensureRule(TableNat, ChainPostRouting, "-j", ChainFabEdgeNatOutgoing); err != nil { - m.log.Error(err, "failed to append rule", "table", TableNat, "chain", ChainPostRouting, "rule", fmt.Sprintf("-j %s", ChainFabEdgeNatOutgoing)) - continue + if err := c.helper.Apply(); err != nil { + m.log.Error(err, "failed to sync iptables rules") + } else { + m.log.V(5).Info("iptables rules is synced") } } @@ -176,19 +149,6 @@ func (m *Manager) areSubnetsEqual(sa1, sa2 []string) bool { return true } -func ensureChain(ipt *iptables.IPTables, table, chain string) error { - exists, err := ipt.ChainExists(table, chain) - if err != nil { - return err - } - - if exists { - return nil - } - - return ipt.NewChain(table, chain) -} - func (m *Manager) getAllPeerCIDRs() (cidrSet4, cidrSet6 sets.String) { cidrSet4, cidrSet6 = sets.NewString(), sets.NewString() diff --git a/pkg/agent/manager.go b/pkg/agent/manager.go index bc0eebb..1352756 100644 --- a/pkg/agent/manager.go +++ b/pkg/agent/manager.go @@ -24,7 +24,6 @@ import ( "sync" "time" - "github.com/coreos/go-iptables/iptables" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" @@ -37,15 +36,8 @@ import ( ) const ( - TableFilter = "filter" - TableNat = "nat" - ChainForward = "FORWARD" - ChainPostRouting = "POSTROUTING" - ChainMasquerade = "MASQUERADE" - ChainFabEdgeForward = "FABEDGE-FORWARD" - ChainFabEdgeNatOutgoing = "FABEDGE-NAT-OUTGOING" - IPSetFabEdgePeerCIDR = "FABEDGE-PEER-CIDR" - IPSetFabEdgePeerCIDR6 = "FABEDGE-PEER-CIDR6" + IPSetFabEdgePeerCIDR = "FABEDGE-PEER-CIDR" + IPSetFabEdgePeerCIDR6 = "FABEDGE-PEER-CIDR6" ) type Manager struct { @@ -54,10 +46,8 @@ type Manager struct { ipvs ipvs.Interface ipset ipset.Interface - tm tunnel.Manager - ipt4 *iptables.IPTables - ipt6 *iptables.IPTables - log logr.Logger + tm tunnel.Manager + log logr.Logger currentEndpoint Endpoint mediatorEndpoint *Endpoint diff --git a/pkg/cloud-agent/cloud_agent.go b/pkg/cloud-agent/cloud_agent.go index c4ed289..6ad9d71 100644 --- a/pkg/cloud-agent/cloud_agent.go +++ b/pkg/cloud-agent/cloud_agent.go @@ -19,13 +19,10 @@ import ( "fmt" "net" "os" - "strings" "sync" "time" "github.com/bep/debounce" - "github.com/coreos/go-iptables/iptables" - netutil "github.com/fabedge/fabedge/pkg/util/net" flag "github.com/spf13/pflag" "github.com/vishvananda/netlink" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -110,15 +107,8 @@ func Execute() { } func NewCloudAgent() (*CloudAgent, error) { - iph, err := newIptableHandler(iptables.ProtocolIPv4) - if err != nil { - return nil, err - } - - iph6, err := newIptableHandler(iptables.ProtocolIPv6) - if err != nil { - return nil, err - } + iph, _ := newIptableHandler() + iph6, _ := newIp6tableHandler() if iph == nil && iph6 == nil { return nil, fmt.Errorf("at lease one iptablesHandler is required") @@ -149,10 +139,14 @@ func (a *CloudAgent) addAndSaveRoutes(cp routing.ConnectorPrefixes) { } routes := a.syncRoutes(cp.LocalPrefixes, cp.RemotePrefixes) + routes = append(routes, a.syncRoutes(cp.LocalPrefixes6, cp.RemotePrefixes6)...) - whitelist := sets.NewString(cp.RemotePrefixes...) - whitelist.Insert(cp.RemotePrefixes6...) + whitelist := sets.NewString() + for _, route := range routes { + whitelist.Insert(route.Dst.String()) + } + if err := routeutil.PurgeStrongSwanRoutes(routeutil.NewDstWhitelist(whitelist)); err != nil { logger.Error(err, "failed to purge stale routes in strongswan table") } @@ -182,18 +176,7 @@ func (a *CloudAgent) syncRoutes(localPrefixes []string, remotePrefixes []string) var routes []netlink.Route for _, rp := range remotePrefixes { - var dst *net.IPNet - - // turn IP to CIDR format - if !strings.Contains(rp, "/") { - if netutil.IsIPv4String(rp) { - rp = rp + "/32" - } else { - rp = rp + "/128" - } - } - - _, dst, err = net.ParseCIDR(rp) + _, dst, err := net.ParseCIDR(rp) if err != nil { logger.Error(err, "failed to parse a remote prefix", "remotePrefix", rp) continue diff --git a/pkg/cloud-agent/iptables.go b/pkg/cloud-agent/iptables.go index f2d6d05..e5d1d1f 100644 --- a/pkg/cloud-agent/iptables.go +++ b/pkg/cloud-agent/iptables.go @@ -15,136 +15,90 @@ package cloud_agent import ( - "fmt" - - "github.com/coreos/go-iptables/iptables" + "bytes" + "github.com/fabedge/fabedge/pkg/util/ipset" ipsetutil "github.com/fabedge/fabedge/pkg/util/ipset" + "github.com/fabedge/fabedge/pkg/util/iptables" "k8s.io/apimachinery/pkg/util/sets" -) - -const ( - TableFilter = "filter" - TableNat = "nat" - ChainForward = "FORWARD" - ChainPostRouting = "POSTROUTING" - ChainFabEdgeForward = "FABEDGE-FORWARD" - ChainFabEdgePostRouting = "FABEDGE-POSTROUTING" - IPSetRemotePodCIDR = "FABEDGE-REMOTE-POD-CIDR" - IPSetRemotePodCIDR6 = "FABEDGE-REMOTE-POD-CIDR6" + "text/template" ) type IptablesHandler struct { - ipt *iptables.IPTables ipset ipsetutil.Interface ipsetName string hashFamily string + helper iptables.ApplierCleaner + rulesData []byte } -func newIptableHandler(version iptables.Protocol) (*IptablesHandler, error) { - var ( - ipsetName string - hashFamily string - ) - - switch version { - case iptables.ProtocolIPv4: - ipsetName, hashFamily = IPSetRemotePodCIDR, ipsetutil.ProtocolFamilyIPV4 - case iptables.ProtocolIPv6: - ipsetName, hashFamily = IPSetRemotePodCIDR6, ipsetutil.ProtocolFamilyIPV6 - default: - return nil, fmt.Errorf("unknown version") - } - - ipt, err := iptables.NewWithProtocol(version) - if err != nil { +func newIptableHandler() (*IptablesHandler, error) { + names := ipset.Names4 + rulesData := bytes.NewBuffer(nil) + if err := tmpl.Execute(rulesData, names); err != nil { return nil, err } return &IptablesHandler{ - ipt: ipt, ipset: ipsetutil.New(), - ipsetName: ipsetName, - hashFamily: hashFamily, + ipsetName: ipset.IPSetRemotePodCIDR, + hashFamily: ipsetutil.ProtocolFamilyIPV4, + helper: iptables.NewApplierCleaner(iptables.ProtocolIPv4, jumpChains, rulesData.Bytes()), + rulesData: rulesData.Bytes(), }, nil } -func (h IptablesHandler) maintainRules(remotePodCIDRs []string) { - if err := h.syncRemotePodCIDRSet(remotePodCIDRs); err != nil { - logger.Error(err, "failed to sync ipset", "setName", h.ipsetName, "remotePodCIDRs", remotePodCIDRs) - } else { - logger.V(5).Info("ipset is synced", "setName", h.ipsetName, "remotePodCIDRs", remotePodCIDRs) - } - - if err := h.syncForwardRules(); err != nil { - logger.Error(err, "failed to sync iptables forward chain") - } else { - logger.V(5).Info("iptables forward chain is synced") +func newIp6tableHandler() (*IptablesHandler, error) { + names := ipset.Names6 + rulesData := bytes.NewBuffer(nil) + if err := tmpl.Execute(rulesData, names); err != nil { + return nil, err } - if err := h.syncPostRoutingRules(); err != nil { - logger.Error(err, "failed to sync iptables post-routing chain") - } else { - logger.V(5).Info("iptables post-routing chain is synced") - } + return &IptablesHandler{ + ipset: ipsetutil.New(), + ipsetName: ipset.IPSetRemotePodCIDR6, + hashFamily: ipsetutil.ProtocolFamilyIPV6, + helper: iptables.NewApplierCleaner(iptables.ProtocolIPv6, jumpChains, rulesData.Bytes()), + rulesData: rulesData.Bytes(), + }, nil } -func (h IptablesHandler) syncForwardRules() (err error) { - if err = h.ipt.ClearChain(TableFilter, ChainFabEdgeForward); err != nil { - return err - } - exists, err := h.ipt.Exists(TableFilter, ChainForward, "-j", ChainFabEdgeForward) - if err != nil { - return err - } - - if !exists { - if err = h.ipt.Insert(TableFilter, ChainForward, 1, "-j", ChainFabEdgeForward); err != nil { - return err - } - } +var tmpl = template.Must(template.New("iptables").Parse(` +*filter +:FABEDGE-FORWARD - [0:0] - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeForward, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil { - return err - } +-A FABEDGE-FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A FABEDGE-FORWARD -m set --match-set {{ .RemotePodCIDR }} src -j ACCEPT +-A FABEDGE-FORWARD -m set --match-set {{ .RemotePodCIDR }} dst -j ACCEPT +COMMIT - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeForward, "-m", "set", "--match-set", h.ipsetName, "src", "-j", "ACCEPT"); err != nil { - return err - } +*nat +:FABEDGE-POSTROUTING - [0:0] +:KUBE-POSTROUTING - [0:0] - if err = h.ipt.AppendUnique(TableFilter, ChainFabEdgeForward, "-m", "set", "--match-set", h.ipsetName, "dst", "-j", "ACCEPT"); err != nil { - return err - } +-A FABEDGE-POSTROUTING -m mark --mark 0x4000/0x4000 -j KUBE-POSTROUTING +-A FABEDGE-POSTROUTING -m set --match-set {{ .RemotePodCIDR }} dst -j ACCEPT +-A FABEDGE-POSTROUTING -m set --match-set {{ .RemotePodCIDR }} src -j ACCEPT +COMMIT +`)) - return nil +var jumpChains = []iptables.JumpChain{ + {Table: iptables.TableFilter, SrcChain: iptables.ChainForward, DstChain: iptables.ChainFabEdgeForward, Position: iptables.Append}, + {Table: iptables.TableNat, SrcChain: iptables.ChainPostRouting, DstChain: iptables.ChainFabEdgePostRouting, Position: iptables.Prepend}, } -func (h IptablesHandler) syncPostRoutingRules() (err error) { - if err = h.ipt.ClearChain(TableNat, ChainFabEdgePostRouting); err != nil { - return err - } - exists, err := h.ipt.Exists(TableNat, ChainPostRouting, "-j", ChainFabEdgePostRouting) - if err != nil { - return err - } - - if !exists { - if err = h.ipt.Insert(TableNat, ChainPostRouting, 1, "-j", ChainFabEdgePostRouting); err != nil { - return err - } - } - - // If packets have 0x4000/0x4000 mark, then traffic should be handled by KUBE-POSTROUTING chain, - // otherwise traffic to nodePort service, sometimes load balancer service, won't be masqueraded, - // and this would cause response packets are dropped - if err = h.ipt.AppendUnique(TableNat, ChainFabEdgePostRouting, "-m", "mark", "--mark", "0x4000/0x4000", "-j", "KUBE-POSTROUTING"); err != nil { - return err +func (h IptablesHandler) maintainRules(remotePodCIDRs []string) { + if err := h.syncRemotePodCIDRSet(remotePodCIDRs); err != nil { + logger.Error(err, "failed to sync ipset", "setName", h.ipsetName, "remotePodCIDRs", remotePodCIDRs) + } else { + logger.V(5).Info("ipset is synced", "setName", h.ipsetName, "remotePodCIDRs", remotePodCIDRs) } - if err = h.ipt.AppendUnique(TableNat, ChainFabEdgePostRouting, "-m", "set", "--match-set", h.ipsetName, "dst", "-j", "ACCEPT"); err != nil { - return err + if err := h.helper.Apply(); err != nil { + logger.Error(err, "failed to sync iptables rules") + } else { + logger.V(5).Info("iptables rules is synced") } - - return h.ipt.AppendUnique(TableNat, ChainFabEdgePostRouting, "-m", "set", "--match-set", h.ipsetName, "src", "-j", "ACCEPT") } func (h IptablesHandler) syncRemotePodCIDRSet(remotePodCIDRs []string) error { @@ -158,9 +112,5 @@ func (h IptablesHandler) syncRemotePodCIDRSet(remotePodCIDRs []string) error { } func (h IptablesHandler) clearRules() error { - if err := h.ipt.ClearChain(TableNat, ChainFabEdgePostRouting); err != nil { - return err - } - - return h.ipt.ClearChain(TableFilter, ChainFabEdgeForward) + return h.helper.Remove() } diff --git a/pkg/connector/iptables.go b/pkg/connector/iptables.go index a8c5667..2aa5c9d 100644 --- a/pkg/connector/iptables.go +++ b/pkg/connector/iptables.go @@ -57,7 +57,7 @@ COMMIT var jumpChains = []iptables.JumpChain{ {Table: iptables.TableFilter, SrcChain: iptables.ChainInput, DstChain: iptables.ChainFabEdgeInput, Position: iptables.Append}, {Table: iptables.TableFilter, SrcChain: iptables.ChainForward, DstChain: iptables.ChainFabEdgeForward, Position: iptables.Append}, - {Table: iptables.TableNAT, SrcChain: iptables.ChainPostRouting, DstChain: iptables.ChainFabEdgePostRouting, Position: iptables.Prepend}, + {Table: iptables.TableNat, SrcChain: iptables.ChainPostRouting, DstChain: iptables.ChainFabEdgePostRouting, Position: iptables.Prepend}, } type IPSetSpec struct { diff --git a/pkg/connector/manager.go b/pkg/connector/manager.go index df6ff00..fedbe92 100644 --- a/pkg/connector/manager.go +++ b/pkg/connector/manager.go @@ -297,6 +297,7 @@ func (m *Manager) broadcastConnectorPrefixes() { } log := m.log.WithValues("connectorPrefixes", cp) + log.V(5).Info("get connector prefixes") b, err := json.Marshal(cp) if err != nil { @@ -331,7 +332,6 @@ func (m *Manager) workLoop() { m.iptHandler.maintainIPTables() m.ipt6Handler.maintainIPTables() - m.broadcastConnectorPrefixes() } } diff --git a/pkg/util/ipset/ipset.go b/pkg/util/ipset/ipset.go index d38c8ea..a573b24 100644 --- a/pkg/util/ipset/ipset.go +++ b/pkg/util/ipset/ipset.go @@ -45,6 +45,8 @@ const ( IPSetCloudPodCIDR6 = "FABEDGE-CLOUD-POD-CIDR6" IPSetCloudNodeCIDR = "FABEDGE-CLOUD-NODE-CIDR" IPSetCloudNodeCIDR6 = "FABEDGE-CLOUD-NODE-CIDR6" + IPSetRemotePodCIDR = "FABEDGE-REMOTE-POD-CIDR" + IPSetRemotePodCIDR6 = "FABEDGE-REMOTE-POD-CIDR6" ) type IPSetNames struct { @@ -52,6 +54,7 @@ type IPSetNames struct { EdgeNodeCIDR string CloudPodCIDR string CloudNodeCIDR string + RemotePodCIDR string } var ( @@ -60,6 +63,7 @@ var ( EdgePodCIDR: IPSetEdgePodCIDR, CloudPodCIDR: IPSetCloudPodCIDR, CloudNodeCIDR: IPSetCloudNodeCIDR, + RemotePodCIDR: IPSetRemotePodCIDR, } Names6 = IPSetNames{ @@ -67,6 +71,7 @@ var ( EdgePodCIDR: IPSetEdgePodCIDR6, CloudPodCIDR: IPSetCloudPodCIDR6, CloudNodeCIDR: IPSetCloudNodeCIDR6, + RemotePodCIDR: IPSetRemotePodCIDR6, } ) diff --git a/pkg/util/iptables/iptables.go b/pkg/util/iptables/iptables.go index d6ed5e7..664b449 100644 --- a/pkg/util/iptables/iptables.go +++ b/pkg/util/iptables/iptables.go @@ -20,40 +20,40 @@ import ( utilexec "k8s.io/utils/exec" ) +type Protocol = utiliptables.Protocol + +const ( + ProtocolIPv4 Protocol = "IPv4" + ProtocolIPv6 Protocol = "IPv6" +) + type Table = utiliptables.Table -// iptables tables const ( - TableFilter utiliptables.Table = "filter" - TableNAT = "nat" + TableFilter Table = "filter" + TableNat Table = "nat" ) type Chain = utiliptables.Chain -// iptables chains const ( - ChainInput utiliptables.Chain = "INPUT" - ChainForward = "FORWARD" - ChainPostRouting = "POSTROUTING" - - ChainFabEdgeInput = "FABEDGE-INPUT" - ChainFabEdgeForward = "FABEDGE-FORWARD" - ChainFabEdgePostRouting = "FABEDGE-POSTROUTING" - ChainFabEdgeNatOutgoing = "FABEDGE-NAT-OUTGOING" + ChainInput Chain = "INPUT" + ChainForward Chain = "FORWARD" + ChainPostRouting Chain = "POSTROUTING" ) -type RulePosition = utiliptables.RulePosition - const ( - Append RulePosition = utiliptables.Append - Prepend = utiliptables.Prepend + ChainFabEdgeInput Chain = "FABEDGE-INPUT" + ChainFabEdgeForward Chain = "FABEDGE-FORWARD" + ChainFabEdgePostRouting Chain = "FABEDGE-POSTROUTING" + ChainFabEdgeNatOutgoing Chain = "FABEDGE-NAT-OUTGOING" ) -type Protocol = utiliptables.Protocol +type RulePosition = utiliptables.RulePosition const ( - ProtocolIPv4 Protocol = "IPv4" - ProtocolIPv6 Protocol = "IPv6" + Append RulePosition = utiliptables.Append + Prepend RulePosition = utiliptables.Prepend ) type FlushFlag = utiliptables.FlushFlag