From 42203353d5f0d73768b317048c5e5f6c3c3124ec Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Wed, 24 Oct 2018 13:37:11 -0400 Subject: [PATCH 1/4] temporary fix for /32 CIDR networking issue Signed-off-by: Brandon Lum --- nabla-lib/network/network_linux.go | 46 ++++++++++++++++++------------ runnc-cont/rumprun.go | 10 ++++++- runnc-cont/runnc_cont.go | 2 +- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/nabla-lib/network/network_linux.go b/nabla-lib/network/network_linux.go index 5a7ae85..ab9897a 100644 --- a/nabla-lib/network/network_linux.go +++ b/nabla-lib/network/network_linux.go @@ -251,13 +251,13 @@ func CreateMacvtapInterfaceDocker(tapName *string, master string) ( return masterIP, gwAddr, masterMask, tapMac, nil } -func getMasterDetails(masterLink netlink.Link) (masterAddr *netlink.Addr, masterIP net.IP, masterMask net.IPMask, gwAddr net.IP, err error) { +func getMasterDetails(masterLink netlink.Link) (masterAddr *netlink.Addr, masterIP net.IP, masterMask net.IPMask, gwAddr net.IP, mac string, err error) { addrs, err := netlink.AddrList(masterLink, netlink.FAMILY_V4) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, "", err } if len(addrs) == 0 { - return nil, nil, nil, nil, fmt.Errorf("master should have an IP") + return nil, nil, nil, nil, "", fmt.Errorf("master should have an IP") } masterAddr = &addrs[0] masterIP = addrs[0].IPNet.IP @@ -265,16 +265,17 @@ func getMasterDetails(masterLink netlink.Link) (masterAddr *netlink.Addr, master routes, err := netlink.RouteList(masterLink, netlink.FAMILY_V4) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, "", err } if len(routes) == 0 { - return nil, nil, nil, nil, + return nil, nil, nil, nil, "", fmt.Errorf("master should have at least one route") } // XXX: is the "gateway" always the first route? gwAddr = routes[0].Gw - return masterAddr, masterIP, masterMask, gwAddr, nil + macAddr := masterLink.Attrs().HardwareAddr.String() + return masterAddr, masterIP, masterMask, gwAddr, macAddr, nil } // CreateTapInterfaceDocker creates a new TAP interface and a bridge, adds both @@ -282,21 +283,21 @@ func getMasterDetails(masterLink netlink.Link) (masterAddr *netlink.Addr, master // of the master link to be used by the unikernel NIC. Returns the assigned // IP/mask and gateway IP. func CreateTapInterfaceDocker(tapName string, master string) ( - net.IP, net.IP, net.IPMask, error) { + net.IP, net.IP, net.IPMask, string, error) { masterLink, err := netlink.LinkByName(master) if err != nil { - return nil, nil, nil, + return nil, nil, nil, "", fmt.Errorf("no master interface: %v", err) } - masterAddr, masterIP, masterMask, gwAddr, err := getMasterDetails(masterLink) + masterAddr, masterIP, masterMask, gwAddr, mac, err := getMasterDetails(masterLink) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, "", err } err = SetupTunDev() if err != nil { - return nil, nil, nil, err + return nil, nil, nil, "", err } // ip tuntap add tap100 mode tap @@ -305,24 +306,34 @@ func CreateTapInterfaceDocker(tapName string, master string) ( Mode: netlink.TUNTAP_MODE_TAP} err = netlink.LinkAdd(tap) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, "", err } // ip link set dev tap100 up' err = netlink.LinkSetUp(tap) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, "", err } // ip addr del $INET_STR dev master err = netlink.AddrDel(masterLink, masterAddr) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, "", err + } + + genmac, err := net.ParseMAC("aa:aa:aa:aa:bb:cc") + if err != nil { + return nil, nil, nil, "", err + } + + err = netlink.LinkSetHardwareAddr(masterLink, genmac) + if err != nil { + return nil, nil, nil, "", err } br0, err := CreateBridge("br0") if err != nil { - return nil, nil, nil, err + return nil, nil, nil, "", err } netlink.LinkSetMaster(masterLink, br0) @@ -331,10 +342,9 @@ func CreateTapInterfaceDocker(tapName string, master string) ( // ip link set dev br0 up' err = netlink.LinkSetUp(br0) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, "", err } - - return masterIP, gwAddr, masterMask, nil + return masterIP, gwAddr, masterMask, mac, nil } // SetupTunDev sets up the /dev/net/tun device if it doesn't exists diff --git a/runnc-cont/rumprun.go b/runnc-cont/rumprun.go index 5f73ea2..a3192d7 100644 --- a/runnc-cont/rumprun.go +++ b/runnc-cont/rumprun.go @@ -105,13 +105,21 @@ func CreateRumprunArgs(ip net.IP, mask net.IPMask, gw net.IP, mountPoint string, envVars []string, cwd string, unikernel string, cmdargs string) (string, error) { + // XXX: Due to bug in: https://github.com/nabla-containers/runnc/issues/40 + // If we detect a /32 mask, we set it to 1 as a "fix", and hope we are in + // the same subnet... (working on a fix for mask:0) + cidr := strconv.Itoa(network.MaskCIDR(mask)) + if cidr == "32" { + cidr = "1" + } + net := rumpArgsNetwork{ If: "ukvmif0", Cloner: "True", Type: "inet", Method: "static", Addr: ip.String(), - Mask: strconv.Itoa(network.MaskCIDR(mask)), + Mask: cidr, Gw: gw.String(), } diff --git a/runnc-cont/runnc_cont.go b/runnc-cont/runnc_cont.go index 593dcc4..5c2ebf2 100644 --- a/runnc-cont/runnc_cont.go +++ b/runnc-cont/runnc_cont.go @@ -142,7 +142,7 @@ func run(nablarun string, unikernel string, tapName string, // container veth pair. // XXX: This is a workaround due to an error with MacvTap, error was : // Could not create /dev/tap8863: open /sys/devices/virtual/net/macvtap8863/tap8863/dev: no such file or directory - ip, gw, mask, err = network.CreateTapInterfaceDocker(tapName, "eth0") + ip, gw, mask, mac, err = network.CreateTapInterfaceDocker(tapName, "eth0") if err != nil { fmt.Fprintf(os.Stderr, "Could not create %s: %v\n", tapName, err) return 1 From 7abdfaf22d8fa8b390fa58f02e7f8fd3c63b9e66 Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Wed, 24 Oct 2018 15:01:49 -0400 Subject: [PATCH 2/4] Updated golang to 1.11 Signed-off-by: Brandon Lum --- .travis.yml | 2 +- Dockerfile.build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6cd778..ae3198c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ language: go go: - - 1.8.x + - 1.11.x - tip matrix: diff --git a/Dockerfile.build b/Dockerfile.build index 64894e8..c0398a4 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -1,4 +1,4 @@ -FROM golang:1.9 +FROM golang:1.11 RUN go get -u github.com/golang/dep/cmd/dep RUN apt update RUN apt install -y genisoimage From b13362b3451d314fbf6e3f91949beff4850dde0e Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Thu, 25 Oct 2018 16:00:28 -0400 Subject: [PATCH 3/4] Added limitations warning to README.md Signed-off-by: Brandon Lum --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c113b86..d9fe603 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,9 @@ sudo docker run --rm --runtime=runnc nablact/nabla-node-base:latest There are many. Some are fixable and being worked on, some are fixable but harder and will take some time, and some others are ones that we don't really know how to fix (or possibly not worth fixing). +Container runtime limitations: +- Unable to properly handle /32 IP address assignments. Current hack converts cidr from 32 to 1 + Here are some missing features that we are currently working on: - a golang base image - MirageOS and IncludeOS base images From fab7db1bddf55091db0ccbcdb8dbf1ff8b870a07 Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Fri, 26 Oct 2018 16:14:30 -0400 Subject: [PATCH 4/4] Added warning for /32 cidr trick Signed-off-by: Brandon Lum --- runnc-cont/rumprun.go | 1 + 1 file changed, 1 insertion(+) diff --git a/runnc-cont/rumprun.go b/runnc-cont/rumprun.go index a3192d7..291def2 100644 --- a/runnc-cont/rumprun.go +++ b/runnc-cont/rumprun.go @@ -110,6 +110,7 @@ func CreateRumprunArgs(ip net.IP, mask net.IPMask, gw net.IP, // the same subnet... (working on a fix for mask:0) cidr := strconv.Itoa(network.MaskCIDR(mask)) if cidr == "32" { + fmt.Printf("WARNING: Changing CIDR from 32 to 1 due to Issue https://github.com/nabla-containers/runnc/issues/40\n") cidr = "1" }