From f8fbb428942f348fb83d1115c052f8ec5d1c2686 Mon Sep 17 00:00:00 2001 From: Murali Reddy Date: Wed, 26 Apr 2017 11:51:30 +0530 Subject: [PATCH] enable kube-route to be run as daemonset, now just run kubectl create -f kube-router-daemonset.yaml --- Dockerfile | 5 ++ Makefile | 12 +++-- README.md | 11 ++--- app/controllers/network_routes_controller.go | 43 ++++++++++------- .../network_services_controller.go | 30 ++++++++---- app/options/options.go | 11 +++-- app/server.go | 7 +++ kube-router-daemonset.yaml | 44 ++++++++++++++++++ utils/pod_cidr.go | 46 +++++++++++++++++-- 9 files changed, 162 insertions(+), 47 deletions(-) create mode 100644 Dockerfile create mode 100644 kube-router-daemonset.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..4bccc56f4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine +RUN apk add --no-cache iptables ipset +COPY kube-router / + +ENTRYPOINT ["/kube-router"] diff --git a/Makefile b/Makefile index a0ccd676e..b845c9ef3 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,13 @@ -#all: push -all: - go build -o kube-router kube-router.go +all: dockerimg + +dockerimg: build + sudo docker build -t "cloudnativelabs/kube-router" . + +build: + go build --ldflags '-extldflags "-static"' -o kube-router kube-router.go clean: rm -f kube-router run: - ./kube-router --kubeconfig=~/kubeconfig + ./kube-router --kubeconfig=/var/lib/kube-router/kubeconfig diff --git a/README.md b/README.md index cb332285a..6a18beb28 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ kube-router ========== -Kube-router is a distributed load balancer, firewall and router for Kubernetes. Kube-router can be configured to provide on each node: +Kube-router is a distributed load balancer, firewall and router for Kubernetes. Kube-router can be configured to provide on each cluster node: - an ingress firewall for the pods running on the node as per the defined Kubernetes network policies - a service proxy on each node for *ClusterIP* and *NodePort* service types, providing service discovery and load balancing @@ -17,13 +17,12 @@ We have Kube-proxy which provides service proxy and load balancer. We have sever ## Theory of Operation -Kube-router runs as agent on each node and leverages standard Linux technologies **iptables, ipvs/lvs, ipset, iproute2** +Kube-router can be run as a agent or a pod (through daemonset) on each node and leverages standard Linux technologies **iptables, ipvs/lvs, ipset, iproute2** ### service proxy and load balancing -Kube-router uses IPVS/LVS technology built in Linux to provide L4 load balancing. Each of the kubernetes service of **ClusterIP** and **NodePort** -type is configured as IPVS virtual service. Each service endpoint is configured as real server to the virtual service. -Standard **ipvsadm** tool can be used to verify the configuration and monitor the status. +Kube-router uses IPVS/LVS technology built in Linux to provide L4 load balancing. Each of the kubernetes service of **ClusterIP** and **NodePort** type is configured as IPVS virtual service. Each service endpoint is configured as real server to the virtual service. +Standard **ipvsadm** tool can be used to verify the configuration and monitor the active connections. Below is example set of services on kubernetes @@ -100,7 +99,7 @@ Alternatively you can download the prebuilt binary from https://github.com/cloud --run-router If true each node advertise routes the rest of the nodes and learn the routes for the pods. false by default --run-service-proxy If false, kube-router won't setup IPVS for services proxy. true by default. --cleanup-config If true cleanup iptables rules, ipvs, ipset configuration and exit. - --cni-conf-file string Full path to CNI configuration file. + --masquerade-all SNAT all traffic to cluster IP/node port. False by default --config-sync-period duration How often configuration from the apiserver is refreshed. Must be greater than 0. (default 1m0s) --iptables-sync-period duration The maximum interval of how often iptables rules are refreshed (e.g. '5s', '1m'). Must be greater than 0. (default 1m0s) --ipvs-sync-period duration The maximum interval of how often ipvs config is refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s) diff --git a/app/controllers/network_routes_controller.go b/app/controllers/network_routes_controller.go index ad4326d59..ceb28c899 100644 --- a/app/controllers/network_routes_controller.go +++ b/app/controllers/network_routes_controller.go @@ -5,6 +5,7 @@ import ( "net" "os" "strconv" + "strings" "sync" "time" @@ -27,12 +28,30 @@ type NetworkRoutingController struct { mu sync.Mutex clientset *kubernetes.Clientset bgpServer *gobgp.BgpServer - cniConfFile string syncPeriod time.Duration } func (nrc *NetworkRoutingController) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) { + cidr, err := utils.GetPodCidrFromCniSpec("/etc/cni/net.d/10-kuberouter.conf") + if err != nil { + glog.Errorf("Failed to get pod CIDR from CNI conf file: %s", err.Error()) + } + cidrlen, _ := cidr.Mask.Size() + oldCidr := cidr.IP.String() + "/" + strconv.Itoa(cidrlen) + + currentCidr, err := utils.GetPodCidrFromNodeSpec(nrc.clientset) + if err != nil { + glog.Errorf("Failed to get pod CIDR from node spec: %s", err.Error()) + } + + if len(cidr.IP) == 0 || strings.Compare(oldCidr, currentCidr) != 0 { + err = utils.InsertPodCidrInCniSpec("/etc/cni/net.d/10-kuberouter.conf", currentCidr) + if err != nil { + glog.Errorf("Failed to insert pod CIDR into CNI conf file: %s", err.Error()) + } + } + t := time.NewTicker(nrc.syncPeriod) defer t.Stop() defer wg.Done() @@ -110,17 +129,20 @@ func (nrc *NetworkRoutingController) watchBgpUpdates() { func (nrc *NetworkRoutingController) advertiseRoute() error { - subnet, cidrlen, err := utils.GetPodCidrDetails(nrc.cniConfFile) + cidr, err := utils.GetPodCidrFromNodeSpec(nrc.clientset) if err != nil { return err } + cidrStr := strings.Split(cidr, "/") + subnet := cidrStr[0] + cidrLen, err := strconv.Atoi(cidrStr[1]) attrs := []bgp.PathAttributeInterface{ bgp.NewPathAttributeOrigin(0), bgp.NewPathAttributeNextHop(nrc.nodeIP.String()), bgp.NewPathAttributeAsPath([]bgp.AsPathParamInterface{bgp.NewAs4PathParam(bgp.BGP_ASPATH_ATTR_TYPE_SEQ, []uint32{4000, 400000, 300000, 40001})}), } - glog.Infof("Advertising route: '%s/%s via %s' to peers", subnet, strconv.Itoa(cidrlen), nrc.nodeIP.String()) - if _, err := nrc.bgpServer.AddPath("", []*table.Path{table.NewPath(nil, bgp.NewIPAddrPrefix(uint8(cidrlen), + glog.Infof("Advertising route: '%s/%s via %s' to peers", subnet, strconv.Itoa(cidrLen), nrc.nodeIP.String()) + if _, err := nrc.bgpServer.AddPath("", []*table.Path{table.NewPath(nil, bgp.NewIPAddrPrefix(uint8(cidrLen), subnet), false, attrs, time.Now(), false)}); err != nil { return fmt.Errorf(err.Error()) } @@ -151,19 +173,6 @@ func NewNetworkRoutingController(clientset *kubernetes.Clientset, kubeRouterConf nrc.syncPeriod = kubeRouterConfig.RoutesSyncPeriod nrc.clientset = clientset - nrc.cniConfFile = kubeRouterConfig.CniConfFile - - if kubeRouterConfig.CniConfFile == "" { - panic("Please specify a valid CNF conf file path in the command line parameter --cni-conf-file ") - } - - if _, err := os.Stat(nrc.cniConfFile); os.IsNotExist(err) { - panic("Specified CNI conf file does not exist. Conf file: " + nrc.cniConfFile) - } - _, _, err := utils.GetPodCidrDetails(nrc.cniConfFile) - if err != nil { - panic("Failed to read IPAM conf from the CNI conf file: " + nrc.cniConfFile + " due to " + err.Error()) - } nodeHostName, err := os.Hostname() if err != nil { diff --git a/app/controllers/network_services_controller.go b/app/controllers/network_services_controller.go index 77d2474a4..a42d86bb4 100644 --- a/app/controllers/network_services_controller.go +++ b/app/controllers/network_services_controller.go @@ -47,6 +47,7 @@ type NetworkServicesController struct { endpointsMap endpointsInfoMap podCidr string masqueradeAll bool + client *kubernetes.Clientset } // internal representation of kubernetes service @@ -348,13 +349,18 @@ func ensureMasqueradeIptablesRule(masqueradeAll bool, podCidr string) error { var args []string if masqueradeAll { args = []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "-j", "MASQUERADE"} - } else { + err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", args...) + if err != nil { + return errors.New("Failed to run iptables command" + err.Error()) + } + } + if len(podCidr) > 0 { args = []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "!", "-s", podCidr, "-j", "MASQUERADE"} - } - err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", args...) - if err != nil { - return errors.New("Failed to run iptables command" + err.Error()) + err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", args...) + if err != nil { + return errors.New("Failed to run iptables command" + err.Error()) + } } glog.Infof("Successfully added iptables masqurade rule") return nil @@ -512,15 +518,19 @@ func NewNetworkServicesController(clientset *kubernetes.Clientset, config *optio nsc.serviceMap = make(serviceInfoMap) nsc.endpointsMap = make(endpointsInfoMap) + nsc.client = clientset + + nsc.masqueradeAll = false + if config.MasqueradeAll { + nsc.masqueradeAll = true + } - nsc.masqueradeAll = true if config.RunRouter { - subnet, cidrLen, err := utils.GetPodCidrDetails(config.CniConfFile) + cidr, err := utils.GetPodCidrFromNodeSpec(nsc.client) if err != nil { - return nil, fmt.Errorf("Failed to get pod CIDR details from CNI conf file: %s", err.Error()) + return nil, fmt.Errorf("Failed to get pod CIDR details from Node.spec: %s", err.Error()) } - nsc.masqueradeAll = false - nsc.podCidr = subnet + "/" + strconv.Itoa(cidrLen) + nsc.podCidr = cidr } nodeHostName, err := os.Hostname() diff --git a/app/options/options.go b/app/options/options.go index d5b1cce92..6e75d5bd8 100755 --- a/app/options/options.go +++ b/app/options/options.go @@ -17,7 +17,7 @@ type KubeRouterConfig struct { RunServiceProxy bool RunFirewall bool RunRouter bool - CniConfFile string + MasqueradeAll bool } func NewKubeRouterConfig() *KubeRouterConfig { @@ -25,21 +25,22 @@ func NewKubeRouterConfig() *KubeRouterConfig { IpvsSyncPeriod: 1 * time.Minute, IPTablesSyncPeriod: 1 * time.Minute, RoutesSyncPeriod: 1 * time.Minute, + MasqueradeAll: false, RunServiceProxy: true, RunFirewall: true, RunRouter: false} } func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) { + fs.BoolVar(&s.RunServiceProxy, "run-service-proxy", s.RunServiceProxy, "If false, kube-router wont setup IPVS for services proxy. True by default.") + fs.BoolVar(&s.RunFirewall, "run-firewall", s.RunFirewall, "If false, kube-router wont setup iptables to provide ingress firewall for pods. True by default.") + fs.BoolVar(&s.RunRouter, "run-router", s.RunRouter, "If true each node advertise routes the rest of the nodes and learn the routes for the pods. True by default.") fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization information (the master location is set by the master flag).") fs.BoolVar(&s.CleanupConfig, "cleanup-config", s.CleanupConfig, "If true cleanup iptables rules, ipvs, ipset configuration and exit.") + fs.BoolVar(&s.MasqueradeAll, "masquerade-all", s.MasqueradeAll, "SNAT all traffic to cluster IP/node port. False by default") fs.DurationVar(&s.ConfigSyncPeriod, "config-sync-period", s.ConfigSyncPeriod, "How often configuration from the apiserver is refreshed. Must be greater than 0.") fs.DurationVar(&s.IPTablesSyncPeriod, "iptables-sync-period", s.IPTablesSyncPeriod, "The maximum interval of how often iptables rules are refreshed (e.g. '5s', '1m'). Must be greater than 0.") fs.DurationVar(&s.IpvsSyncPeriod, "ipvs-sync-period", s.IpvsSyncPeriod, "The maximum interval of how often ipvs config is refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") fs.DurationVar(&s.RoutesSyncPeriod, "routes-sync-period", s.RoutesSyncPeriod, "The maximum interval of how often routes are adrvertised and learned (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") - fs.BoolVar(&s.RunServiceProxy, "run-service-proxy", s.RunServiceProxy, "If false, kube-router wont setup IPVS for services proxy. True by default.") - fs.BoolVar(&s.RunFirewall, "run-firewall", s.RunFirewall, "If false, kube-router wont setup iptables to provide ingress firewall for pods. True by default.") - fs.BoolVar(&s.RunRouter, "run-router", s.RunRouter, "If true each node advertise routes the rest of the nodes and learn the routes for the pods. False by default.") - fs.StringVar(&s.CniConfFile, "cni-conf-file", s.CniConfFile, "Full path to CNI configuration file.") } diff --git a/app/server.go b/app/server.go index cb137913b..d1897a5b3 100644 --- a/app/server.go +++ b/app/server.go @@ -22,6 +22,13 @@ type KubeRouter struct { func NewKubeRouterDefault(config *options.KubeRouterConfig) (*KubeRouter, error) { + if len(config.Master) == 0 || len(config.Kubeconfig) == 0 { + if _, err := os.Stat("/var/lib/kube-router/kubeconfig"); os.IsNotExist(err) { + panic("Either one of --master or --kubeconfig must be specified. Or valid kubeconfig file must exist as /var/lib/kube-router/kubeconfig") + } + config.Kubeconfig = "/var/lib/kube-router/kubeconfig" + } + clientconfig, err := clientcmd.BuildConfigFromFlags(config.Master, config.Kubeconfig) if err != nil { panic(err.Error()) diff --git a/kube-router-daemonset.yaml b/kube-router-daemonset.yaml new file mode 100644 index 000000000..0ab2816f6 --- /dev/null +++ b/kube-router-daemonset.yaml @@ -0,0 +1,44 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-router + namespace: kube-system + labels: + app: kube-router +spec: + template: + metadata: + labels: + name: kube-router + spec: + hostNetwork: true + containers: + - name: kube-router + image: cloudnativelabs/kube-router + args: ["--run-router=false"] + securityContext: + privileged: true + imagePullPolicy: Always + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /etc/cni/net.d/10-kuberouter.conf + name: cni-conf-dir + - mountPath: /var/lib/kube-router/kubeconfig + name: kubeconfig + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d/10-kuberouter.conf + - name: kubeconfig + hostPath: + path: /var/lib/kube-router/kubeconfig diff --git a/utils/pod_cidr.go b/utils/pod_cidr.go index df53b54f5..c680fcd89 100644 --- a/utils/pod_cidr.go +++ b/utils/pod_cidr.go @@ -1,24 +1,60 @@ package utils import ( + "encoding/json" "fmt" + "io/ioutil" + "net" + "os" "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/plugins/ipam/host-local/backend/allocator" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" ) -func GetPodCidrDetails(cniConfFilePath string) (string, int, error) { +func GetPodCidrFromCniSpec(cniConfFilePath string) (net.IPNet, error) { netconfig, err := libcni.ConfFromFile(cniConfFilePath) if err != nil { - return "", 0, fmt.Errorf("Failed to load CNI conf: %s", err.Error()) + return net.IPNet{}, fmt.Errorf("Failed to load CNI conf file: %s", err.Error()) } var ipamConfig *allocator.IPAMConfig ipamConfig, _, err = allocator.LoadIPAMConfig(netconfig.Bytes, "") if err != nil { - return "", 0, fmt.Errorf("Failed to get IPAM details from the CNI conf file: %s", err.Error()) + return net.IPNet{}, fmt.Errorf("Failed to get IPAM details from the CNI conf file: %s", err.Error()) } + return net.IPNet(ipamConfig.Subnet), nil +} + +func InsertPodCidrInCniSpec(cniConfFilePath string, cidr string) error { + file, err := ioutil.ReadFile(cniConfFilePath) + if err != nil { + return fmt.Errorf("Failed to load CNI conf file: %s", err.Error()) + } + config := make(map[string]interface{}) + err = json.Unmarshal(file, &config) + if err != nil { + return fmt.Errorf("Failed to parse JSON from CNI conf file: %s", err.Error()) + } + + config["ipam"].(map[string]interface{})["subnet"] = cidr + configJson, _ := json.Marshal(config) + err = ioutil.WriteFile(cniConfFilePath, configJson, 0644) + if err != nil { + return fmt.Errorf("Failed to insert subnet cidr into CNI conf file: %s", err.Error()) + } + return nil +} - cidrlen, _ := ipamConfig.Subnet.Mask.Size() - return ipamConfig.Subnet.IP.String(), cidrlen, nil +func GetPodCidrFromNodeSpec(clientset *kubernetes.Clientset) (string, error) { + nodeHostName, err := os.Hostname() + if err != nil { + panic(err.Error()) + } + node, err := clientset.Core().Nodes().Get(nodeHostName, metav1.GetOptions{}) + if err != nil { + panic(err.Error()) + } + return node.Spec.PodCIDR, nil }