feat: Add supports for dual-stack

build/virt-prerunner/Dockerfile

@@ -11,10 +11,11 @@ RUN go mod download
 COPY cmd/ cmd/
 COPY pkg/ pkg/
 RUN --mount=type=cache,target=/root/.cache/go-build go build -a cmd/virt-prerunner/main.go
+RUN --mount=type=cache,target=/root/.cache/go-build go build -o rad -a cmd/route-advertisement-daemon/main.go
 
-FROM alpine
+FROM alpine:3.17
 
-RUN apk add --no-cache curl screen dnsmasq cdrkit iptables iproute2 qemu-virtiofsd dpkg util-linux s6-overlay nmap-ncat
+RUN apk add --no-cache curl screen dnsmasq kea-dhcp6 cdrkit iptables ip6tables iproute2 qemu-virtiofsd dpkg util-linux s6-overlay nmap-ncat
 
 RUN set -eux; \
     mkdir /var/lib/cloud-hypervisor; \
@@ -40,6 +41,7 @@ COPY build/virt-prerunner/cloud-hypervisor-finish.sh /etc/s6-overlay/s6-rc.d/clo
 RUN touch /etc/s6-overlay/s6-rc.d/user/contents.d/cloud-hypervisor
 
 COPY --from=builder /workspace/main /usr/bin/virt-prerunner
+COPY --from=builder /workspace/rad /usr/bin/rad
 COPY build/virt-prerunner/virt-prerunner-type /etc/s6-overlay/s6-rc.d/virt-prerunner/type
 COPY build/virt-prerunner/virt-prerunner-up /etc/s6-overlay/s6-rc.d/virt-prerunner/up
 COPY build/virt-prerunner/virt-prerunner-run.sh /etc/s6-overlay/scripts/virt-prerunner-run.sh
@@ -50,5 +52,7 @@ ENTRYPOINT ["/init"]
 
 COPY build/virt-prerunner/iptables-wrapper /sbin/iptables-wrapper
 RUN update-alternatives --install /sbin/iptables iptables /sbin/iptables-wrapper 100
+COPY build/virt-prerunner/ip6tables-wrapper /sbin/ip6tables-wrapper
+RUN update-alternatives --install /sbin/ip6tables ip6tables /sbin/ip6tables-wrapper 100
 
 ADD build/virt-prerunner/virt-init-volume.sh /usr/bin/virt-init-volume

build/virt-prerunner/ip6tables-wrapper

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set +e
+
+ip6tables-legacy -nvL
+
+if [ $? -eq 0 ]
+then
+  mode=legacy
+else
+  mode=nft
+fi
+
+update-alternatives --install /sbin/ip6tables ip6tables "/sbin/ip6tables-${mode}" 100
+update-alternatives --set ip6tables "/sbin/ip6tables-${mode}" > /dev/null
+
+exec "$0" "$@"

cmd/route-advertisement-daemon/main.go

@@ -0,0 +1,335 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/netip"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	"github.com/jcelliott/lumber"
+	"github.com/mdlayher/ndp"
+	"golang.org/x/net/ipv6"
+
+	"github.com/smartxworks/virtink/pkg/ipv6util"
+)
+
+var (
+	logFile       string
+	iface         string
+	router        string
+	isRemoteRoute bool
+	client        string
+	clientHWAddr  string
+	prefix        string
+
+	linkLocalAllRouters = netip.MustParseAddr("ff02::2")
+
+	logger *lumber.FileLogger
+)
+
+func main() {
+	if logFile == "" {
+		log.Fatal("the log-file may not be empty")
+	}
+
+	if err := os.MkdirAll(filepath.Dir(logFile), 0755); err != nil {
+		log.Fatalf("create log dir: %s", err)
+	}
+	var err error
+	logger, err = lumber.NewRotateLogger(logFile, 2000, 5)
+	if err != nil {
+		log.Fatalf("create rotate logger: %s", err)
+	}
+	defer logger.Close()
+	logger.Level(lumber.INFO)
+
+	src, dst, cidr, err := validateVars()
+	if err != nil {
+		logger.Error("validate vars: %s", err)
+		return
+	}
+	if src != nil && isRemoteRoute {
+		if _, err := executeCommand("ip", "-6", "neigh", "add", client, "lladdr", clientHWAddr, "dev", iface); err != nil {
+			logger.Error("add neighbor entry for client: %s", err)
+			return
+		}
+
+		ipv6LLA := src
+		if !src.IsLinkLocalUnicast() {
+			mac, err := tryDiscoveryNeighborMAC(iface, src, 5)
+			if err != nil {
+				logger.Error("discovery router MAC: %s", err)
+				return
+			}
+			ipv6LLA = ipv6util.GenerateEUI64Address(net.ParseIP("fe80::0"), mac)
+			mac2, err := tryDiscoveryNeighborMAC(iface, ipv6LLA, 5)
+			if err != nil {
+				logger.Error("discovery router MAC: %s", err)
+				return
+			}
+			if mac.String() != mac2.String() {
+				logger.Error("failed to get router link-local address")
+				return
+			}
+		}
+
+		if _, err := executeCommand("ip6tables", "-A", "OUTPUT", "-o", iface, "--src", ipv6LLA.String(), "-p", "icmpv6", "--icmpv6-type", "neighbor-solicitation", "-j", "DROP"); err != nil {
+			logger.Error("drop neighbor solicitation on interface: %s", err)
+			return
+		}
+		if _, err := executeCommand("ip6tables", "-A", "OUTPUT", "-o", iface, "--src", ipv6LLA.String(), "-p", "icmpv6", "--icmpv6-type", "neighbor-advertisement", "-j", "DROP"); err != nil {
+			logger.Error("drop neighbor advertisement on interface: %s", err)
+			return
+		}
+
+		// As described in RFC 4861 section-4.2, the srouce address of RA must be the link-local
+		// address assigned to the interface from which the message is sent, so the LLA of default
+		// router have to be added to the interface. And the followings need to be done.
+		//  1.Disable DAD of the interface
+		//  2.Add static neighbor entry for the client, otherwise the interface will send a NS
+		//    message with it's MAC in options to client
+		//  3.Drop NA message from interface responsed to NS message learning default router LLA
+		//  4.Drop NS message from interface with default router LLA in options
+		if executeCommand("ip", "addr", "add", fmt.Sprintf("%s/64", ipv6LLA.String()), "dev", iface); err != nil {
+			logger.Error("add IPv6 addr to the interface: %s", err)
+			return
+		}
+
+		src = ipv6LLA
+	}
+
+	if err := startRouteAdvertisement(iface, src, dst, cidr); err != nil {
+		logger.Error("start route advertisement: %s", err)
+		return
+	}
+}
+
+func validateVars() (net.IP, net.IP, *net.IPNet, error) {
+	if iface == "" {
+		return nil, nil, nil, fmt.Errorf("the interface may not be empty")
+	}
+
+	var src net.IP
+	if router != "" {
+		src = net.ParseIP(router)
+		if src == nil {
+			return nil, nil, nil, fmt.Errorf("the router IPv6 address (%s) is illegal", router)
+		}
+		if isRemoteRoute {
+			if clientHWAddr == "" {
+				return nil, nil, nil, fmt.Errorf("the client-hardware-addr may not be empty when router is remote")
+			}
+		}
+	}
+
+	if client == "" {
+		if clientHWAddr == "" {
+			return nil, nil, nil, fmt.Errorf("the client and client-hardware-addr may not both be empty")
+		}
+		clientMAC, err := net.ParseMAC(clientHWAddr)
+		if err != nil {
+			return nil, nil, nil, fmt.Errorf("parse MAC: %s", err)
+		}
+		client = ipv6util.GenerateEUI64Address(net.ParseIP("fe80::0"), clientMAC).String()
+	}
+	dst := net.ParseIP(client)
+	if dst == nil {
+		return nil, nil, nil, fmt.Errorf("the client IPv6 address (%s) is illegal", client)
+	}
+	if !dst.IsLinkLocalUnicast() {
+		return nil, nil, nil, fmt.Errorf("the client IPv6 address should be a link-local address")
+	}
+
+	if prefix == "" {
+		return nil, nil, nil, fmt.Errorf("the prefix may not be empty")
+	}
+	_, cidr, err := net.ParseCIDR(prefix)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("the prefix (%s) is illegal", prefix)
+	}
+
+	return src, dst, cidr, nil
+}
+
+func tryDiscoveryNeighborMAC(ifaceName string, ip net.IP, retry int) (net.HardwareAddr, error) {
+	for i := retry; i > 0; i-- {
+		mac, err := discoveryNeighborMAC(ifaceName, ip)
+		if err != nil {
+			return nil, err
+		}
+		if mac == nil {
+			logger.Info("retry in 5s")
+			continue
+		}
+		return mac, nil
+	}
+
+	return nil, fmt.Errorf("failed to discovery neighbor MAC. Try %d times", retry)
+}
+
+func discoveryNeighborMAC(ifaceName string, ip net.IP) (net.HardwareAddr, error) {
+	iface, err := net.InterfaceByName(ifaceName)
+	if err != nil {
+		return nil, fmt.Errorf("get interface by name: %s", err)
+	}
+	conn, err := tryCreateNDPConn(iface, 5)
+	if err != nil {
+		return nil, fmt.Errorf("create NDP connection: %s", err)
+	}
+	defer conn.Close()
+
+	target := netip.MustParseAddr(ip.String())
+	solicitationAddr, err := ndp.SolicitedNodeMulticast(target)
+	if err != nil {
+		return nil, fmt.Errorf("determine solicited-node multicast address: %s", err)
+	}
+	solicitation := &ndp.NeighborSolicitation{
+		TargetAddress: target,
+		Options: []ndp.Option{
+			&ndp.LinkLayerAddress{
+				Direction: ndp.Source,
+				Addr:      iface.HardwareAddr,
+			},
+		},
+	}
+	if err := conn.WriteTo(solicitation, nil, solicitationAddr); err != nil {
+		return nil, fmt.Errorf("write neighbor solicitation: %s", err)
+	}
+
+	var f ipv6.ICMPFilter
+	f.SetAll(true)
+	f.Accept(ipv6.ICMPTypeNeighborAdvertisement)
+	if err := conn.SetICMPFilter(&f); err != nil {
+		return nil, fmt.Errorf("set ICMPv6 filter: %s", err)
+	}
+
+	msg, _, from, err := conn.ReadFrom()
+	if err != nil {
+		return nil, fmt.Errorf("read NDP message: %s", err)
+	}
+	if target.WithZone(ifaceName).Compare(from) != 0 && target.Compare(from) != 0 {
+		logger.Info("the NDP message is not from solicitation target")
+		return nil, nil
+	}
+	advertisement := msg.(*ndp.NeighborAdvertisement)
+	if len(advertisement.Options) != 1 {
+		return nil, fmt.Errorf("get %d option(s) in neighbor advertisement, but expect one", len(advertisement.Options))
+	}
+	linkLayerAddr, ok := advertisement.Options[0].(*ndp.LinkLayerAddress)
+	if !ok {
+		return nil, fmt.Errorf("advertisement option is not a link-layer address")
+	}
+	return linkLayerAddr.Addr, nil
+}
+
+func tryCreateNDPConn(iface *net.Interface, retry int) (*ndp.Conn, error) {
+	var err error
+	for i := retry; i > 0; i-- {
+		conn, _, err := ndp.Listen(iface, ndp.LinkLocal)
+		if err != nil {
+			// caused by tap device state down?
+			logger.Warn("listen interface link-local address: %s. Retry in 5s", err)
+			time.Sleep(5 * time.Second)
+		}
+		if err == nil {
+			return conn, nil
+		}
+	}
+
+	return nil, fmt.Errorf("listen interface link-local address: %s. Retry %d times", err, retry)
+}
+
+func startRouteAdvertisement(ifaceName string, src net.IP, dst net.IP, cidr *net.IPNet) error {
+	iface, err := net.InterfaceByName(ifaceName)
+	if err != nil {
+		return fmt.Errorf("get interface by name: %s", err)
+	}
+	conn, err := tryCreateNDPConn(iface, 5)
+	if err != nil {
+		return fmt.Errorf("create NDP connection: %s", err)
+	}
+	defer conn.Close()
+
+	var filter ipv6.ICMPFilter
+	filter.SetAll(true)
+	filter.Accept(ipv6.ICMPTypeRouterSolicitation)
+	if err := conn.SetICMPFilter(&filter); err != nil {
+		return fmt.Errorf("apply ICMPv6 filter: %s", err)
+	}
+	if err := conn.JoinGroup(linkLocalAllRouters); err != nil {
+		return fmt.Errorf("join IPv6 link-local all routers multicast group: %s", err)
+	}
+
+	prefixLen, _ := cidr.Mask.Size()
+	advertisement := &ndp.RouterAdvertisement{
+		CurrentHopLimit:      255,
+		RouterLifetime:       65535 * time.Second,
+		ManagedConfiguration: true,
+		OtherConfiguration:   true,
+		Options: []ndp.Option{
+			&ndp.PrefixInformation{
+				PrefixLength:  uint8(prefixLen),
+				Prefix:        netip.MustParseAddr(cidr.IP.String()),
+				OnLink:        true,
+				ValidLifetime: 4294967295 * time.Second,
+			},
+		},
+	}
+
+	controlMsg := &ipv6.ControlMessage{
+		HopLimit: 255,
+		Src:      src,
+	}
+
+	if src == nil {
+		advertisement.RouterLifetime = 0
+		controlMsg = nil
+	}
+
+	for {
+		_, _, from, err := conn.ReadFrom()
+		if err != nil {
+			logger.Warn("read NDP message: %s. Retry in 5s", err)
+			time.Sleep(5 * time.Second)
+			continue
+		}
+		target := netip.MustParseAddr(dst.String())
+		if target.WithZone(ifaceName).Compare(from) != 0 && target.Compare(from) != 0 {
+			continue
+		}
+
+		if err := conn.WriteTo(advertisement, controlMsg, netip.MustParseAddr(dst.String())); err != nil {
+			return fmt.Errorf("send route advertisement: %s", err)
+		}
+		logger.Info("reply RS from %s", dst.String())
+	}
+}
+
+func executeCommand(name string, arg ...string) (string, error) {
+	cmd := exec.Command(name, arg...)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return string(output), fmt.Errorf("%q: %s: %s", cmd.String(), err, output)
+	}
+	return string(output), nil
+}
+
+func init() {
+	flag.StringVar(&logFile, "log-file", "", "The log file to write to.")
+	flag.StringVar(&iface, "interface", "", "The interface to listen to.")
+	flag.StringVar(&router, "router", "", "The IPv6 address of the default router. "+
+		"It's recommanded to use the link-local address of the router, "+
+		"otherwise the SLAAC link-local address formed by router hardware address will be used.")
+	flag.BoolVar(&isRemoteRoute, "is-remote-route", false, "")
+	flag.StringVar(&client, "client", "", "The IPv6 link-local address of the client to advertise to. "+
+		"The SLAAC link-local address formed by client hardware address will be used when empty.")
+	flag.StringVar(&clientHWAddr, "client-hardware-addr", "", "The hardware address of the client.")
+	flag.StringVar(&prefix, "prefix", "", "The prefix of the subnet.")
+
+	flag.Parse()
+}