feat: specify an ovn underlay network through MicroOVN (#361)

closes #239

api/services.go

@@ -41,7 +41,7 @@ func servicesPut(state *state.State, r *http.Request) response.Response {
 	services := make([]types.ServiceType, len(req.Tokens))
 	for i, cfg := range req.Tokens {
 		services[i] = types.ServiceType(cfg.Service)
-		joinConfigs[cfg.Service] = service.JoinConfig{Token: cfg.JoinToken, LXDConfig: req.LXDConfig, CephConfig: req.CephConfig}
+		joinConfigs[cfg.Service] = service.JoinConfig{Token: cfg.JoinToken, LXDConfig: req.LXDConfig, CephConfig: req.CephConfig, OVNConfig: req.OVNConfig}
 	}
 
 	// Default to the first iface if none specified.

api/types/services.go

@@ -29,6 +29,7 @@ type ServicesPut struct {
 
 	LXDConfig  []api.ClusterMemberConfigKey `json:"lxd_config" yaml:"lxd_config"`
 	CephConfig []types.DisksPost            `json:"ceph_config" yaml:"ceph_config"`
+	OVNConfig  map[string]string            `json:"ovn_config" yaml:"ovn_config"`
 }
 
 // ServiceToken represents a join token for a service join request.

cmd/microcloud/ask.go

@@ -481,7 +481,7 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error {
 	return nil
 }
 
-func validateCephInterfacesForSubnet(lxdService *service.LXDService, systems map[string]InitSystem, availableCephNetworkInterfaces map[string]map[string]service.CephDedicatedInterface, askedCephSubnet string) error {
+func validateCephInterfacesForSubnet(lxdService *service.LXDService, systems map[string]InitSystem, availableCephNetworkInterfaces map[string]map[string]service.DedicatedInterface, askedCephSubnet string) error {
 	validatedCephInterfacesData, err := lxdService.ValidateCephInterfaces(askedCephSubnet, availableCephNetworkInterfaces)
 	if err != nil {
 		return err
@@ -1104,6 +1104,76 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
 		}
 	}
 
+	canOVNUnderlay := true
+	for peer, system := range c.systems {
+		if len(c.state[system.ServerInfo.Name].AvailableOVNInterfaces) == 0 {
+			fmt.Printf("Not enough interfaces available on %s to create an underlay network, skipping\n", peer)
+			canOVNUnderlay = false
+			break
+		}
+	}
+
+	var ovnUnderlaySelectedIPs map[string]string
+	ovnUnderlayData := [][]string{}
+	for peer, system := range c.systems {
+		// skip any systems that have already been clustered, but are available for other configuration.
+		state, ok := c.state[c.name]
+		if ok {
+			if state.ExistingServices[types.MicroOVN][peer] != "" {
+				continue
+			}
+		}
+
+		for _, net := range c.state[system.ServerInfo.Name].AvailableOVNInterfaces {
+			for _, addr := range net.Addresses {
+				ovnUnderlayData = append(ovnUnderlayData, []string{peer, net.Network.Name, net.Network.Type, addr})
+			}
+		}
+	}
+
+	if len(ovnUnderlayData) != 0 && canOVNUnderlay {
+		wantsDedicatedUnderlay, err := c.asker.AskBool("Configure dedicated underlay networking? (yes/no) [default=no]: ", "no")
+		if err != nil {
+			return err
+		}
+
+		if wantsDedicatedUnderlay {
+			header = []string{"LOCATION", "IFACE", "TYPE", "IP ADDRESS (CIDR)"}
+			fmt.Println("Select exactly one network interface from each cluster member:")
+
+			table = NewSelectableTable(header, ovnUnderlayData)
+			ovnUnderlaySelectedIPs = map[string]string{}
+			err = c.askRetry("Retry selecting underlay network interfaces?", func() error {
+				err = table.Render(table.rows)
+				if err != nil {
+					return err
+				}
+
+				answers, err := table.GetSelections()
+				if err != nil {
+					return err
+				}
+
+				ovnUnderlaySelectedIPs = map[string]string{}
+				for _, answer := range answers {
+					target := table.SelectionValue(answer, "LOCATION")
+					ipAddr := table.SelectionValue(answer, "IP ADDRESS (CIDR)")
+
+					if ovnUnderlaySelectedIPs[target] != "" {
+						return fmt.Errorf("Failed to configure OVN underlay traffic: Selected more than one interface for target %q", target)
+					}
+
+					ovnUnderlaySelectedIPs[target] = ipAddr
+				}
+
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	for peer, system := range c.systems {
 		if !askSystems[peer] {
 			continue
@@ -1133,6 +1203,19 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
 			system.Networks = append(system.Networks, finalConfigs...)
 		}
 
+		if ovnUnderlaySelectedIPs != nil {
+			ovnUnderlayIpAddr, ok := ovnUnderlaySelectedIPs[peer]
+			if ok {
+				ip, _, err := net.ParseCIDR(ovnUnderlayIpAddr)
+				if err != nil {
+					return err
+				}
+
+				fmt.Printf("Using %q for OVN underlay traffic on %q\n", ip.String(), peer)
+				system.OVNGeneveAddr = ip.String()
+			}
+		}
+
 		c.systems[peer] = system
 	}
 
@@ -1220,15 +1303,15 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error {
 		return nil
 	}
 
-	availableCephNetworkInterfaces := map[string]map[string]service.CephDedicatedInterface{}
+	availableCephNetworkInterfaces := map[string]map[string]service.DedicatedInterface{}
 	for name, state := range c.state {
 		if len(state.AvailableCephInterfaces) == 0 {
 			fmt.Printf("No network interfaces found with IPs on %q to set a dedicated Ceph network, skipping Ceph network setup\n", name)
 
 			return nil
 		}
 
-		ifaces := make(map[string]service.CephDedicatedInterface, len(state.AvailableCephInterfaces))
+		ifaces := make(map[string]service.DedicatedInterface, len(state.AvailableCephInterfaces))
 		for name, iface := range state.AvailableCephInterfaces {
 			ifaces[name] = iface
 		}

cmd/microcloud/main_init.go

@@ -53,6 +53,9 @@ type InitSystem struct {
 	TargetStoragePools []lxdAPI.StoragePoolsPost
 	// Networks is the cluster-wide network configuration.
 	Networks []lxdAPI.NetworksPost
+	// OVNGeneveAddr represents an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system.
+	// If left empty, the system will choose to route the Geneve traffic through the management network.
+	OVNGeneveAddr string
 	// StoragePools is the cluster-wide storage pool configuration.
 	StoragePools []lxdAPI.StoragePoolsPost
 	// StorageVolumes is the cluster-wide storage volume configuration.
@@ -524,6 +527,12 @@ func (c *initConfig) addPeers(sh *service.Handler) (revert.Hook, error) {
 			LXDConfig:  info.JoinConfig,
 			CephConfig: info.MicroCephDisks,
 		}
+
+		if info.OVNGeneveAddr != "" {
+			p := joinConfig[peer]
+			p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveAddr}
+			joinConfig[peer] = p
+		}
 	}
 
 	clusterSize := map[types.ServiceType]int{}
@@ -782,6 +791,17 @@ func (c *initConfig) setupCluster(s *service.Handler) error {
 			}
 		}
 
+		if s.Type() == types.MicroOVN {
+			microOvnBootstrapConf := make(map[string]string)
+			if bootstrapSystem.OVNGeneveAddr != "" {
+				microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveAddr
+			}
+
+			if len(microOvnBootstrapConf) > 0 {
+				s.SetConfig(microOvnBootstrapConf)
+			}
+		}
+
 		// set a 2 minute timeout to bootstrap a service in case the node is slow.
 		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 		defer cancel()

cmd/microcloud/main_init_preseed.go

@@ -38,6 +38,7 @@ type Preseed struct {
 type System struct {
 	Name            string      `yaml:"name"`
 	UplinkInterface string      `yaml:"ovn_uplink_interface"`
+	UnderlayIP      string      `yaml:"underlay_ip"`
 	Storage         InitStorage `yaml:"storage"`
 }
 
@@ -220,6 +221,7 @@ func (c *initConfig) RunPreseed(cmd *cobra.Command) error {
 // validate validates the unmarshaled preseed input.
 func (p *Preseed) validate(name string, bootstrap bool) error {
 	uplinkCount := 0
+	underlayCount := 0
 	directCephCount := 0
 	directLocalCount := 0
 	localInit := false
@@ -241,6 +243,15 @@ func (p *Preseed) validate(name string, bootstrap bool) error {
 			uplinkCount++
 		}
 
+		if system.UnderlayIP != "" {
+			_, _, err := net.ParseCIDR(system.UnderlayIP)
+			if err != nil {
+				return fmt.Errorf("Invalid underlay IP: %w", err)
+			}
+
+			underlayCount++
+		}
+
 		if len(system.Storage.Ceph) > 0 {
 			directCephCount++
 		}
@@ -270,6 +281,11 @@ func (p *Preseed) validate(name string, bootstrap bool) error {
 		return fmt.Errorf("Some systems are missing an uplink interface")
 	}
 
+	containsUnderlay := underlayCount > 0
+	if containsUnderlay && underlayCount < len(p.Systems) {
+		return fmt.Errorf("Some systems are missing an underlay interface")
+	}
+
 	containsLocalStorage = directLocalCount > 0
 	if containsLocalStorage && directLocalCount < len(p.Systems) && len(p.Storage.Local) == 0 {
 		return fmt.Errorf("Some systems are missing local storage disks")
@@ -484,10 +500,15 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste
 
 	lxd := s.Services[types.LXD].(*service.LXDService)
 	ifaceByPeer := map[string]string{}
+	ovnUnderlayNeeded := false
 	for _, cfg := range p.Systems {
 		if cfg.UplinkInterface != "" {
 			ifaceByPeer[cfg.Name] = cfg.UplinkInterface
 		}
+
+		if cfg.UnderlayIP != "" {
+			ovnUnderlayNeeded = true
+		}
 	}
 
 	localInfo, err := s.CollectSystemInformation(context.Background(), mdns.ServerInfo{Name: c.name, Address: c.address})
@@ -498,7 +519,7 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste
 	// If an uplink interface was explicitly chosen, we will try to set up an OVN network.
 	explicitOVN := len(ifaceByPeer) > 0
 
-	cephInterfaces := map[string]map[string]service.CephDedicatedInterface{}
+	cephInterfaces := map[string]map[string]service.DedicatedInterface{}
 	for _, system := range c.systems {
 		uplinkIfaces, cephIfaces, _, err := lxd.GetNetworkInterfaces(context.Background(), system.ServerInfo.Name, system.ServerInfo.Address, system.ServerInfo.AuthSecret)
 		if err != nil {
@@ -517,7 +538,7 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste
 
 		for ifaceName, iface := range cephIfaces {
 			if cephInterfaces[system.ServerInfo.Name] == nil {
-				cephInterfaces[system.ServerInfo.Name] = map[string]service.CephDedicatedInterface{}
+				cephInterfaces[system.ServerInfo.Name] = map[string]service.DedicatedInterface{}
 			}
 
 			cephInterfaces[system.ServerInfo.Name][ifaceName] = iface
@@ -542,6 +563,58 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste
 
 			c.systems[peer] = system
 		}
+
+		// Check the preseed underlay network configuration against the available ips.
+		if ovnUnderlayNeeded {
+			canOVNUnderlay := true
+			for peer, system := range c.state {
+				if len(system.AvailableOVNInterfaces) == 0 {
+					fmt.Printf("Not enough interfaces available on %s to create an underlay network, skipping\n", peer)
+					canOVNUnderlay = false
+					break
+				}
+			}
+
+			if canOVNUnderlay {
+				// TODO: call `s.Services[types.MicroOVN].(*service.OVNService).SupportsFeature(context.Background(), "custom_encapsulation_ip")`
+				// when MicroCloud will be updated with microcluster/v2
+				underlays := make(map[string]string, len(p.Systems))
+				for _, sys := range p.Systems {
+					underlays[sys.Name] = sys.UnderlayIP
+				}
+
+				underlayCount := 0
+				for _, sys := range p.Systems {
+					for _, net := range c.state[sys.Name].AvailableOVNInterfaces {
+						if len(net.Addresses) != 0 {
+							for _, cidrAddr := range net.Addresses {
+								if underlays[sys.Name] == cidrAddr {
+									underlayCount = underlayCount + 1
+									goto out
+								}
+							}
+						}
+					}
+
+				out:
+				}
+
+				if underlayCount != len(p.Systems) {
+					return nil, fmt.Errorf("Failed to find all underlay IPs on the network")
+				}
+
+				// Apply the underlay IPs to the systems.
+				for peer, system := range c.systems {
+					ip, _, err := net.ParseCIDR(underlays[peer])
+					if err != nil {
+						return nil, fmt.Errorf("Failed to parse underlay IP: %w", err)
+					}
+
+					system.OVNGeneveAddr = ip.String()
+					c.systems[peer] = system
+				}
+			}
+		}
 	} else {
 		// Check if FAN networking is usable.
 		fanUsable, _, err := service.FanNetworkUsable()

doc/explanation/microcloud.md

@@ -61,6 +61,19 @@ MicroCloud will still be usable, but you will see some limitations:
   As a result of this, network forwarding works at a basic level only, and external addresses must be forwarded to a specific cluster member and don't fail over.
 - There is no support for hardware acceleration, load balancers, or ACL functionality within the local network.
 
+(microcloud-networking-underlay)=
+
+Network interface to connect to an OVN underlay network
+:  While setting up the distributed networking with OVN, you can also decide to set up a dedicated underlay network for the OVN traffic.
+   A dedicated underlay network for OVN serves as the physical infrastructure over which the virtual networking (overlay network) is constructed. It can be beneficial to set up a dedicated underlay network for the following reasons:
+
+- A dedicated underlay network isolates the overlay network traffic from other types of traffic (such as management traffic).
+- By dedicating a physical network solely for the purpose of the virtual network overlay, network congestion can be reduced.
+- A dedicated underlay network can be tuned to provide predictable latency and bandwidth characteristics, which are crucial for the performance of latency-sensitive applications running over the OVN overlay.
+- A dedicated underlay can be designed to scale independently of other networks, which allows for more efficient scaling of the overlay network as the demand for virtual network resources increases.
+
+See {ref}`howto-ovn-underlay` for how to set up a dedicated OVN underlay network.
+
 ### Dedicated internal network for Ceph
 You can set up a dedicated network for Ceph to separate the internal Ceph traffic from the rest of the MicroCloud cluster traffic.
 

doc/how-to/index.md

@@ -10,6 +10,7 @@ Install MicroCloud </how-to/install>
 Manage the snaps </how-to/snaps>
 Initialise MicroCloud </how-to/initialise>
 Configure Ceph networking </how-to/ceph_networking>
+Configure OVN underlay </how-to/ovn_underlay>
 Add a machine </how-to/add_machine>
 Remove a machine </how-to/remove_machine>
 Add a service </how-to/add_service>

doc/how-to/initialise.md

@@ -90,6 +90,13 @@ Complete the following steps to initialise MicroCloud:
       You must select one network interface per machine.
    1. If you want to use IPv4, specify the IPv4 gateway on the uplink network (in CIDR notation) and the first and last IPv4 address in the range that you want to use with LXD.
    1. If you want to use IPv6, specify the IPv6 gateway on the uplink network (in CIDR notation).
+   1. If you chose to set up distributed networking, you can choose to setup an underlay network for the distributed networking:
+
+      If you choose ``yes``, configure the underlay network:
+
+      1. Select the network interfaces that you want to use (see {ref}`microcloud-networking-underlay`).
+
+         You must select one network interface with an IP address per machine.
 1. MicroCloud now starts to bootstrap the cluster.
    Monitor the output to see whether all steps complete successfully.
    See {ref}`bootstrapping-process` for more information.