Skip to content

Commit

Permalink
Add option to auto add nodeCIDR & serviceCIDR route table
Browse files Browse the repository at this point in the history
  • Loading branch information
STARRY-S committed Jul 30, 2024
1 parent 75dc1a9 commit fe5cb37
Show file tree
Hide file tree
Showing 17 changed files with 273 additions and 164 deletions.
6 changes: 2 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,8 @@ Environment variables for operator:
- `CATTLE_ELECTION_RENEW_DEADLINE`: leader election renew deadline, default `30s`.
- `CATTLE_ELECTION_RETRY_PERIOD`: leader election retry period, default `2s`.
- `FLAT_NETWORK_CNI_ARP_POLICY`: CNI ARP Policy, default `arp_notify`, available `arp_notify`, `arping`.

### CNI

> WIP
- `FLAT_NETWORK_CLUSTER_CIDR`: Kubernetes config Cluster CIDR, default `10.42.0.0/16`.
- `FLAT_NETWORK_SERVICE_CIDR`: Kubernetes config Service CIDR, default `10.43.0.0/16`.

## License

Expand Down
19 changes: 15 additions & 4 deletions charts/rancher-flat-network/questions.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
rancher_min_version: 2.9.0-0
categories:
- Networking
namespace: cattle-flat-network
Expand All @@ -8,12 +7,12 @@ questions:
default: "Default"
type: enum
label: "Cluster Type"
group: "Basic Setup"
group: "CNI Plugin"
options:
- "Default"
- "K3s"
- variable: multus.cni.version
description: "CNI Version (K3s '1.0.0', RKE2 '0.3.1')"
description: "CNI Version (K3s is '1.0.0', RKE2 is '0.3.1')"
default: "1.0.0"
type: enum
label: "CNI Version"
Expand All @@ -37,14 +36,26 @@ questions:
type: boolean
label: "Proxy ARP"
group: "CNI Plugin"
- variable: clusterCIDR
default: "10.42.0.0/16"
description: "Kubernetes config Cluster CIDR"
type: string
label: "Cluster CIDR"
group: "CNI Plugin"
- variable: serviceCIDR
default: "10.43.0.0/16"
description: "Kubernetes config Service CIDR"
type: string
label: "Service CIDR"
group: "CNI Plugin"
- variable: flatNetworkOperator.limits.memory
default: "512Mi"
description: "Memory limit for Operator pod"
type: string
label: FlatNetwork Operator
group: "Resource Limits"
- variable: flatNetworkOperator.limits.cpu
default: "100m"
default: "200m"
description: "CPU limit for Operator pod"
type: string
label: FlatNetwork Operator
Expand Down
7 changes: 7 additions & 0 deletions charts/rancher-flat-network/templates/crds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ spec:
gateway:
nullable: true
type: string
ipvlanMode:
nullable: true
type: string
master:
nullable: true
type: string
Expand All @@ -118,10 +121,14 @@ spec:
type: array
routeSettings:
properties:
addClusterCIDR:
type: boolean
addNodeCIDR:
type: boolean
addPodIPToHost:
type: boolean
addServiceCIDR:
type: boolean
flatNetworkDefaultGateway:
type: boolean
type: object
Expand Down
6 changes: 6 additions & 0 deletions charts/rancher-flat-network/templates/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ spec:
value: {{ .Values.flatNetworkOperator.cattleResyncDefault | quote }}
- name: CATTLE_DEV_MODE
value: {{ .Values.flatNetworkOperator.cattleDevMode | quote }}
- name: FLAT_NETWORK_CNI_ARP_POLICY
value: {{ .Values.arpPolicy | quote }}
- name: FLAT_NETWORK_CLUSTER_CIDR
value: {{ .Values.clusterCIDR | quote }}
- name: FLAT_NETWORK_SERVICE_CIDR
value: {{ .Values.serviceCIDR | quote }}
resources:
limits:
memory: {{ .Values.flatNetworkOperator.limits.memory | quote }}
Expand Down
5 changes: 5 additions & 0 deletions charts/rancher-flat-network/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@ proxyARP: false
# Set to 'K3s' if using K3s cluster.
clusterType: default

# Kubernetes config Cluster CIDR
clusterCIDR: "10.42.0.0/16"
# Kubernetes config Service CIDR
serviceCIDR: "10.43.0.0/16"

# Configuration for rancher-flat-network-operator
flatNetworkOperator:
replicas: 2
Expand Down
1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ require (
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
Expand Down
24 changes: 19 additions & 5 deletions pkg/apis/flatnetwork.pandaria.io/v1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,11 @@ type SubnetSpec struct {
// ipvlan: 'l2, l3, l3s' (default 'l2');
Mode string `json:"mode"`

// IPvlanMode is the mode of IPvlan.
// Only required when Mode is 'ipvlan'.
// can be 'bridge, private, vepa' (default 'bridge')
IPvlanMode string `json:"ipvlanMode"`

// Gateway is the gateway of the subnet (optional).
Gateway net.IP `json:"gateway"`

Expand All @@ -116,7 +121,7 @@ type SubnetSpec struct {
Routes []Route `json:"routes,omitempty"`

// RouteSettings provides some advanced options for custom routes.
RouteSettings RouteSettings `json:"routeSettings,oitempty"`
RouteSettings RouteSettings `json:"routeSettings"`
}

type SubnetStatus struct {
Expand Down Expand Up @@ -144,17 +149,26 @@ type Route struct {
}

type RouteSettings struct {
// AddNodeCIDR adds node CIDR route for flat-network pod if enabled.
// AddClusterCIDR adds ClusterCIDR route to eth0 for flat-network pod if enabled.
// This option can be used when eth1 (flatNetwork iface) is default gateway.
AddClusterCIDR bool `json:"addClusterCIDR"`

// AddServiceCIDR adds ServiceCIDR route to eth0 for flat-network pod if enabled.
// This option can be used when eth1 (flatNetwork iface) is default gateway.
AddServiceCIDR bool `json:"addServiceCIDR"`

// AddNodeCIDR adds node CIDR route to eth0 for flat-network pod if enabled.
// This option can be used when eth1 (flatNetwork iface) is default gateway.
AddNodeCIDR bool `json:"addNodeCIDR"`

// AddPodIPToHost adds pod flat-network IP routes on host if enabled.
// AddPodIPToHost adds pod flat-network IP routes on node host NS if enabled.
// If true, it will allow node to directly access Pods running on the current node by flat-network IP.
// If false, node cannot access Pods running on the current node by flat-network IP.
AddPodIPToHost bool `json:"addPodIPToHost"`

// FlatNetworkDefaultGateway lets Pod using the flat-network iface as default gateway.
// NOTE: need to add custom routes (serviceCIDR, clusterCIDR, nodeCIDR)
// if pod using the flat-network iface as the default gateway.
// NOTE: set 'addClusterCIDR', 'addServiceCIDR', 'addNodeCIDR' to true if needed
// when pod is using the flat-network iface as the default gateway.
//
// And the pods’ access to other networks will be restricted.
// For example, Pods cannot directly access the public networks.
Expand Down
26 changes: 23 additions & 3 deletions pkg/cni/commands/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -251,12 +251,12 @@ func Add(args *skel.CmdArgs) error {
ipc.Interface = types100.Int(0)
}
err = netns.Do(func(_ ns.NetNS) error {
if n.FlatNetworkConfig.RuntimeConfig.ARPPolicy == arpNotifyPolicy {
if n.FlatNetworkConfig.ARPPolicy == arpNotifyPolicy {
logrus.Debugf("setting up sysctl arp_notify: %s", args.IfName)
_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv4/conf/%s/arp_notify", args.IfName), "1")
}

if n.FlatNetworkConfig.RuntimeConfig.ProxyARP {
if n.FlatNetworkConfig.ProxyARP {
logrus.Debugf("setting up sysctl proxy_arp: %s", args.IfName)
_, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv4/conf/%s/proxy_arp", args.IfName), "1")
}
Expand All @@ -266,7 +266,7 @@ func Add(args *skel.CmdArgs) error {
err, args.IfName, result)
}

if n.FlatNetworkConfig.RuntimeConfig.ARPPolicy == arpingPolicy {
if n.FlatNetworkConfig.ARPPolicy == arpingPolicy {
logrus.Debugf("sending arping request: %s", args.IfName)
contVeth, err := net.InterfaceByName(args.IfName)
if err != nil {
Expand All @@ -289,6 +289,26 @@ func Add(args *skel.CmdArgs) error {

result.DNS = n.DNS

// Add ClusterCIDR route in Pod NS
if subnet.Spec.RouteSettings.AddClusterCIDR {
logrus.Debugf("adding kube config clusterCIDR %q route to pod NS",
n.FlatNetworkConfig.ClusterCIDR)
err = route.AddPodKubeCIDRRoutes(netns, n.FlatNetworkConfig.ClusterCIDR)
if err != nil {
return fmt.Errorf("failed to add ClusterCIDR route: %w", err)
}
}

// Add ServiceCIDR route in Pod NS
if subnet.Spec.RouteSettings.AddServiceCIDR {
logrus.Debugf("adding kube config serviceCIDR %q route to pod NS",
n.FlatNetworkConfig.ServiceCIDR)
err = route.AddPodKubeCIDRRoutes(netns, n.FlatNetworkConfig.ServiceCIDR)
if err != nil {
return fmt.Errorf("failed to add ServiceCIDR route: %w", err)
}
}

// Add NodeCIDR route in Pod NS
if subnet.Spec.RouteSettings.AddNodeCIDR {
err = route.AddPodNodeCIDRRoutes(netns)
Expand Down
1 change: 1 addition & 0 deletions pkg/cni/logger/logger.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ const (
// Setup logrus loglevel and output file.
// Output logfile to 'logFileFormat' only when 'loggingFlagFile' exists.
func Setup() error {
logrus.SetOutput(io.Discard) // Discard log output by default
logrus.SetFormatter(&nested.Formatter{
HideKeys: false,
TimestampFormat: time.DateTime,
Expand Down
59 changes: 59 additions & 0 deletions pkg/cni/route/kube_cidr.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package route

import (
"fmt"
"net"

"github.com/containernetworking/plugins/pkg/ns"
"github.com/vishvananda/netlink"
)

func AddPodKubeCIDRRoutes(podNS ns.NetNS, cidr string) error {
ip, network, err := net.ParseCIDR(cidr)
if err != nil {
return fmt.Errorf("failed to parse CIDR %q: %w", cidr, err)
}
err = podNS.Do(func(_ ns.NetNS) error {
defaultLinkSet, err := GetDefaultLinkIDSet()
if err != nil {
return fmt.Errorf("failed to get pod default link id: %w", err)
}

podDefaultRoutes, err := GetDefaultRoutes()
if err != nil {
return fmt.Errorf("failed to get pod default routes: %w", err)
}
if len(podDefaultRoutes) == 0 {
return nil
}
var podDefaultGatewayV4 net.IP
var podDefaultGatewayV6 net.IP
for _, r := range podDefaultRoutes {
switch r.Family {
case netlink.FAMILY_V4:
podDefaultGatewayV4 = r.Gw
default:
podDefaultGatewayV6 = r.Gw
}
}

for id := range defaultLinkSet {
r := netlink.Route{
LinkIndex: id,
Dst: network,
Family: netlink.FAMILY_V4,
Gw: podDefaultGatewayV4,
}
if ip.To16() != nil && len(ip.To4()) == 0 {
r.Family = netlink.FAMILY_V6
r.Gw = podDefaultGatewayV6
}
return EnsureRouteExists(&r)
}
return nil
})
if err != nil {
return fmt.Errorf("addPodKubeCIDRRoutes: %w", err)
}
return nil
}
93 changes: 93 additions & 0 deletions pkg/cni/route/node.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package route

import (
"fmt"
"net"

flv1 "github.com/cnrancher/rancher-flat-network/pkg/apis/flatnetwork.pandaria.io/v1"
"github.com/cnrancher/rancher-flat-network/pkg/cni/common"
"github.com/cnrancher/rancher-flat-network/pkg/utils"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netlink/nl"
)

// getHostCIDRCustomRoutes for adding host iface IP addr routes to pod
func getHostCIDRCustomRoutes(linkID int, gwV4, gwV6 net.IP) ([]flv1.Route, error) {
link, err := netlink.LinkByIndex(linkID)
if err != nil {
return nil, fmt.Errorf("getHostCIDRCustomRoutes: %w", err)
}
addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
if err != nil {
return nil, fmt.Errorf("getHostCIDRCustomRoutes: %w", err)
}
if len(addrs) == 0 {
return nil, nil
}
routes := []flv1.Route{}
for _, a := range addrs {
if a.IP.IsLinkLocalUnicast() {
continue
}
r := flv1.Route{
Dev: common.PodIfaceEth0,
Dst: a.IPNet.String(),
Via: nil,
}
switch nl.GetIPFamily(a.IP) {
case netlink.FAMILY_V4:
r.Via = gwV4
default:
r.Via = gwV6
}
routes = append(routes, r)
}
logrus.Debugf("getHostCIDRCustomRoutes: %v", utils.Print(routes))
return routes, nil
}

func AddPodNodeCIDRRoutes(podNS ns.NetNS) error {
// Add host iface IP addr routes and user custom routes to Pod
customRoutes := []flv1.Route{}
defaultLinkSet, err := GetDefaultLinkIDSet()
if err != nil {
return fmt.Errorf("failed to get pod default link id: %w", err)
}

var podDefaultGatewayV4 net.IP
var podDefaultGatewayV6 net.IP
if err := podNS.Do(func(_ ns.NetNS) error {
podDefaultRoutes, err := GetDefaultRoutes()
if err != nil {
return fmt.Errorf("failed to get pod default routes: %w", err)
}
if len(podDefaultRoutes) == 0 {
return nil
}
for _, r := range podDefaultRoutes {
switch r.Family {
case netlink.FAMILY_V4:
podDefaultGatewayV4 = r.Gw
default:
podDefaultGatewayV6 = r.Gw
}
}
return nil
}); err != nil {
return fmt.Errorf("addPodNodeCIDRRoutes: %w", err)
}
for id := range defaultLinkSet {
results, err := getHostCIDRCustomRoutes(id, podDefaultGatewayV4, podDefaultGatewayV6)
if err != nil {
return fmt.Errorf("addPodNodeCIDRRoutes: %w", err)
}
if len(results) == 0 {
continue
}
customRoutes = append(customRoutes, results...)
}

return AddPodFlatNetworkCustomRoutes(podNS, customRoutes)
}
Loading

0 comments on commit fe5cb37

Please sign in to comment.