From d8f50ce37e726590c3717b526618e060037a3dff Mon Sep 17 00:00:00 2001
From: Gabriel Mougard <gabriel.mougard@canonical.com>
Date: Wed, 10 Jul 2024 14:48:19 +0200
Subject: [PATCH 01/10] refactor: Update MicroOVN dep

Signed-off-by: Gabriel Mougard <gabriel.mougard@canonical.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index c757dde7f..8346faf94 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/canonical/lxd v0.0.0-20240709081608-9df996e36bac
 	github.com/canonical/microceph/microceph v0.0.0-20240708123233-b17a96f27a3d
 	github.com/canonical/microcluster v0.0.0-20240709163503-3d587817830c
-	github.com/canonical/microovn/microovn v0.0.0-20240703103351-546b832b3167
+	github.com/canonical/microovn/microovn v0.0.0-20240710084353-a14a37d36234
 	github.com/creack/pty v1.1.21
 	github.com/gorilla/mux v1.8.1
 	github.com/hashicorp/mdns v1.0.5
diff --git a/go.sum b/go.sum
index e82f1b56f..532b4fb9a 100644
--- a/go.sum
+++ b/go.sum
@@ -64,8 +64,8 @@ github.com/canonical/microceph/microceph v0.0.0-20240708123233-b17a96f27a3d h1:s
 github.com/canonical/microceph/microceph v0.0.0-20240708123233-b17a96f27a3d/go.mod h1:bfL5e3XQSSoWlwNDVTnBlu8+38kRIV7kMNvhioxFBQ4=
 github.com/canonical/microcluster v0.0.0-20240709163503-3d587817830c h1:Kjz8OZ2rFcpGfF9KYSJ6iAWraK131IQlO/13VB+saQ0=
 github.com/canonical/microcluster v0.0.0-20240709163503-3d587817830c/go.mod h1:ldDJ32PQhoc7sTSkQnUs2CQPnjUOkkXrzlbUaq9EIPI=
-github.com/canonical/microovn/microovn v0.0.0-20240703103351-546b832b3167 h1:rEMSn67Nd68PEDAt0GE/tUEQpOGEwfuY0eyDR0TBMWo=
-github.com/canonical/microovn/microovn v0.0.0-20240703103351-546b832b3167/go.mod h1:d0NHxbKZTvNjxAurMFNkAT/aCkBEg4owKB2HjNXZaMQ=
+github.com/canonical/microovn/microovn v0.0.0-20240710084353-a14a37d36234 h1:jLP6bhGfzajUtNty6peVqHGrldRlDdgA2seeDIedCqI=
+github.com/canonical/microovn/microovn v0.0.0-20240710084353-a14a37d36234/go.mod h1:SBLx+SJM/I9ZoQCipdOBLC94MqvO5DJdTLrm4sGeW9k=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=

From dd9dbad20364885c14d523a063e70290eb9f3ae9 Mon Sep 17 00:00:00 2001
From: Gabriel Mougard <gabriel.mougard@canonical.com>
Date: Wed, 3 Jul 2024 18:58:16 +0200
Subject: [PATCH 02/10] cmd/microcloud: Add `OVNGeneveAddr` system parameter

Signed-off-by: Gabriel Mougard <gabriel.mougard@canonical.com>
---
 cmd/microcloud/main_init.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go
index 8dab2fe0e..4c99520b4 100644
--- a/cmd/microcloud/main_init.go
+++ b/cmd/microcloud/main_init.go
@@ -53,6 +53,9 @@ type InitSystem struct {
 	TargetStoragePools []lxdAPI.StoragePoolsPost
 	// Networks is the cluster-wide network configuration.
 	Networks []lxdAPI.NetworksPost
+	// OVNGeneveAddr represents an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system.
+	// If left empty, the system will choose to route the Geneve traffic through the management network.
+	OVNGeneveAddr string
 	// StoragePools is the cluster-wide storage pool configuration.
 	StoragePools []lxdAPI.StoragePoolsPost
 	// StorageVolumes is the cluster-wide storage volume configuration.

From b7732575372601012babf7995689a27430fb53ad Mon Sep 17 00:00:00 2001
From: Gabriel Mougard <gabriel.mougard@canonical.com>
Date: Wed, 31 Jul 2024 16:45:36 +0200
Subject: [PATCH 03/10] service/lxd: Merge dedicated Ceph and OVN dedicated
 interfaces in one result

Signed-off-by: Gabriel Mougard <gabriel.mougard@canonical.com>
---
 cmd/microcloud/ask.go               |  6 +++---
 cmd/microcloud/main_init_preseed.go |  4 ++--
 service/lxd.go                      | 25 ++++++++++++++++---------
 service/system_information.go       | 14 +++++++++++---
 4 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go
index 87441de8e..40b4d81e2 100644
--- a/cmd/microcloud/ask.go
+++ b/cmd/microcloud/ask.go
@@ -481,7 +481,7 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error {
 	return nil
 }
 
-func validateCephInterfacesForSubnet(lxdService *service.LXDService, systems map[string]InitSystem, availableCephNetworkInterfaces map[string]map[string]service.CephDedicatedInterface, askedCephSubnet string) error {
+func validateCephInterfacesForSubnet(lxdService *service.LXDService, systems map[string]InitSystem, availableCephNetworkInterfaces map[string]map[string]service.DedicatedInterface, askedCephSubnet string) error {
 	validatedCephInterfacesData, err := lxdService.ValidateCephInterfaces(askedCephSubnet, availableCephNetworkInterfaces)
 	if err != nil {
 		return err
@@ -1220,7 +1220,7 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error {
 		return nil
 	}
 
-	availableCephNetworkInterfaces := map[string]map[string]service.CephDedicatedInterface{}
+	availableCephNetworkInterfaces := map[string]map[string]service.DedicatedInterface{}
 	for name, state := range c.state {
 		if len(state.AvailableCephInterfaces) == 0 {
 			fmt.Printf("No network interfaces found with IPs on %q to set a dedicated Ceph network, skipping Ceph network setup\n", name)
@@ -1228,7 +1228,7 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error {
 			return nil
 		}
 
-		ifaces := make(map[string]service.CephDedicatedInterface, len(state.AvailableCephInterfaces))
+		ifaces := make(map[string]service.DedicatedInterface, len(state.AvailableCephInterfaces))
 		for name, iface := range state.AvailableCephInterfaces {
 			ifaces[name] = iface
 		}
diff --git a/cmd/microcloud/main_init_preseed.go b/cmd/microcloud/main_init_preseed.go
index 61b998593..411bc04c7 100644
--- a/cmd/microcloud/main_init_preseed.go
+++ b/cmd/microcloud/main_init_preseed.go
@@ -498,7 +498,7 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste
 	// If an uplink interface was explicitly chosen, we will try to set up an OVN network.
 	explicitOVN := len(ifaceByPeer) > 0
 
-	cephInterfaces := map[string]map[string]service.CephDedicatedInterface{}
+	cephInterfaces := map[string]map[string]service.DedicatedInterface{}
 	for _, system := range c.systems {
 		uplinkIfaces, cephIfaces, _, err := lxd.GetNetworkInterfaces(context.Background(), system.ServerInfo.Name, system.ServerInfo.Address, system.ServerInfo.AuthSecret)
 		if err != nil {
@@ -517,7 +517,7 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste
 
 		for ifaceName, iface := range cephIfaces {
 			if cephInterfaces[system.ServerInfo.Name] == nil {
-				cephInterfaces[system.ServerInfo.Name] = map[string]service.CephDedicatedInterface{}
+				cephInterfaces[system.ServerInfo.Name] = map[string]service.DedicatedInterface{}
 			}
 
 			cephInterfaces[system.ServerInfo.Name][ifaceName] = iface
diff --git a/service/lxd.go b/service/lxd.go
index d4e858fac..9e962f85c 100644
--- a/service/lxd.go
+++ b/service/lxd.go
@@ -518,24 +518,26 @@ func defaultNetworkInterfacesFilter(client lxd.InstanceServer, network api.Netwo
 				continue
 			}
 
-			addresses = append(addresses, address.Address)
+			addresses = append(addresses, fmt.Sprintf("%s/%s", address.Address, address.Netmask))
 		}
 	}
 
 	return false, addresses
 }
 
-// CephDedicatedInterface represents a dedicated interface for Ceph.
-type CephDedicatedInterface struct {
+// DedicatedInterface represents a dedicated interface for OVN.
+type DedicatedInterface struct {
 	Type      string
+	Network   api.Network
 	Addresses []string
 }
 
 // GetNetworkInterfaces fetches the list of networks from LXD and returns the following:
 // - A map of uplink compatible networks keyed by interface name.
 // - A map of ceph compatible networks keyed by interface name.
+// - A map of ovn compatible networks keyed by interface name.
 // - The list of all networks.
-func (s LXDService) GetNetworkInterfaces(ctx context.Context, name string, address string, secret string) (map[string]api.Network, map[string]CephDedicatedInterface, []api.Network, error) {
+func (s LXDService) GetNetworkInterfaces(ctx context.Context, name string, address string, secret string) (map[string]api.Network, map[string]DedicatedInterface, []api.Network, error) {
 	var err error
 	var client lxd.InstanceServer
 	if name == s.Name() {
@@ -554,7 +556,7 @@ func (s LXDService) GetNetworkInterfaces(ctx context.Context, name string, addre
 	}
 
 	uplinkInterfaces := map[string]api.Network{}
-	cephInterfaces := map[string]CephDedicatedInterface{}
+	dedicatedInterfaces := map[string]DedicatedInterface{}
 	for _, network := range networks {
 		filtered, addresses := defaultNetworkInterfacesFilter(client, network)
 		if filtered {
@@ -564,19 +566,20 @@ func (s LXDService) GetNetworkInterfaces(ctx context.Context, name string, addre
 		if len(addresses) == 0 {
 			uplinkInterfaces[network.Name] = network
 		} else {
-			cephInterfaces[network.Name] = CephDedicatedInterface{
+			dedicatedInterfaces[network.Name] = DedicatedInterface{
 				Type:      network.Type,
+				Network:   network,
 				Addresses: addresses,
 			}
 		}
 	}
 
-	return uplinkInterfaces, cephInterfaces, networks, nil
+	return uplinkInterfaces, dedicatedInterfaces, networks, nil
 }
 
 // ValidateCephInterfaces validates the given interfaces map against the given Ceph network subnet
 // and returns a map of peer name to interfaces that are in the subnet.
-func (s *LXDService) ValidateCephInterfaces(cephNetworkSubnetStr string, peerInterfaces map[string]map[string]CephDedicatedInterface) (map[string][][]string, error) {
+func (s *LXDService) ValidateCephInterfaces(cephNetworkSubnetStr string, peerInterfaces map[string]map[string]DedicatedInterface) (map[string][][]string, error) {
 	_, subnet, err := net.ParseCIDR(cephNetworkSubnetStr)
 	if err != nil {
 		return nil, fmt.Errorf("Invalid CIDR subnet: %v", err)
@@ -593,7 +596,11 @@ func (s *LXDService) ValidateCephInterfaces(cephNetworkSubnetStr string, peerInt
 			for _, addr := range iface.Addresses {
 				ip := net.ParseIP(addr)
 				if ip == nil {
-					return nil, fmt.Errorf("Invalid IP address: %v", addr)
+					// Attempt to parse the IP address as a CIDR.
+					ip, _, err = net.ParseCIDR(addr)
+					if err != nil {
+						return nil, fmt.Errorf("Could not parse either IP nor CIDR notation for address %q: %v", addr, err)
+					}
 				}
 
 				if (subnet.IP.To4() != nil && ip.To4() != nil && subnet.Contains(ip)) || (subnet.IP.To16() != nil && ip.To16() != nil && subnet.Contains(ip)) {
diff --git a/service/system_information.go b/service/system_information.go
index d18eb9d0c..7c57dee8c 100644
--- a/service/system_information.go
+++ b/service/system_information.go
@@ -33,7 +33,10 @@ type SystemInformation struct {
 	AvailableUplinkInterfaces map[string]api.Network
 
 	// AvailableCephInterfaces is the list of networks that can be used for the Ceph cluster network.
-	AvailableCephInterfaces map[string]CephDedicatedInterface
+	AvailableCephInterfaces map[string]DedicatedInterface
+
+	// AvailableOVNInterfaces is the list of networks that can be used for an OVN underlay network.
+	AvailableOVNInterfaces map[string]DedicatedInterface
 
 	// LXDLocalConfig is the local configuration of LXD on this system.
 	LXDLocalConfig map[string]any
@@ -78,7 +81,8 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mdn
 		AuthSecret:                connectInfo.AuthSecret,
 		AvailableDisks:            map[string]api.ResourcesStorageDisk{},
 		AvailableUplinkInterfaces: map[string]api.Network{},
-		AvailableCephInterfaces:   map[string]CephDedicatedInterface{},
+		AvailableCephInterfaces:   map[string]DedicatedInterface{},
+		AvailableOVNInterfaces:    map[string]DedicatedInterface{},
 	}
 
 	var err error
@@ -108,11 +112,15 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mdn
 	}
 
 	var allNets []api.Network
-	s.AvailableUplinkInterfaces, s.AvailableCephInterfaces, allNets, err = lxd.GetNetworkInterfaces(ctx, s.ClusterName, s.ClusterAddress, s.AuthSecret)
+	uplinkInterfaces, dedicatedInterfaces, allNets, err := lxd.GetNetworkInterfaces(ctx, s.ClusterName, s.ClusterAddress, s.AuthSecret)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to get network interfaces on %q: %w", s.ClusterName, err)
 	}
 
+	s.AvailableUplinkInterfaces = uplinkInterfaces
+	s.AvailableCephInterfaces = dedicatedInterfaces
+	s.AvailableOVNInterfaces = dedicatedInterfaces
+
 	for _, network := range allNets {
 		if network.Name == DefaultFANNetwork {
 			s.existingFanNetwork = &network

From 83ae28a743c6ba4f751eeb7e871e46bbd7e0a4b5 Mon Sep 17 00:00:00 2001
From: Gabriel Mougard <gabriel.mougard@canonical.com>
Date: Wed, 31 Jul 2024 16:55:48 +0200
Subject: [PATCH 04/10] service: Implement new `SupportsFeature` function

This will be needed to check the API extensions of different services from MicroCloud

Signed-off-by: Gabriel Mougard <gabriel.mougard@canonical.com>
---
 service/interface.go  |  1 +
 service/lxd.go        | 19 +++++++++++++++++++
 service/microceph.go  | 16 ++++++++++++++++
 service/microcloud.go | 16 ++++++++++++++++
 service/microovn.go   | 16 ++++++++++++++++
 5 files changed, 68 insertions(+)

diff --git a/service/interface.go b/service/interface.go
index cfa045ab4..aa90d585a 100644
--- a/service/interface.go
+++ b/service/interface.go
@@ -24,4 +24,5 @@ type Service interface {
 	Address() string
 	Port() int64
 	SetConfig(config map[string]string)
+	SupportsFeature(ctx context.Context, feature string) (bool, error)
 }
diff --git a/service/lxd.go b/service/lxd.go
index 9e962f85c..ad8edbab9 100644
--- a/service/lxd.go
+++ b/service/lxd.go
@@ -783,3 +783,22 @@ func (s LXDService) defaultGatewaySubnetV4() (*net.IPNet, string, error) {
 
 	return subnet, ifaceName, nil
 }
+
+// SupportsFeature checks if the specified API feature of this Service instance if supported.
+func (s LXDService) SupportsFeature(ctx context.Context, feature string) (bool, error) {
+	c, err := s.Client(ctx, "")
+	if err != nil {
+		return false, err
+	}
+
+	server, _, err := c.GetServer()
+	if err != nil {
+		return false, err
+	}
+
+	if server.APIExtensions == nil {
+		return false, fmt.Errorf("API extensions not available when checking for a LXD feature")
+	}
+
+	return shared.ValueInSlice(feature, server.APIExtensions), nil
+}
diff --git a/service/microceph.go b/service/microceph.go
index 0da37716c..555021586 100644
--- a/service/microceph.go
+++ b/service/microceph.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/canonical/lxd/lxd/util"
 	"github.com/canonical/lxd/shared"
+	"github.com/canonical/lxd/shared/logger"
 	cephTypes "github.com/canonical/microceph/microceph/api/types"
 	cephClient "github.com/canonical/microceph/microceph/client"
 	"github.com/canonical/microcluster/client"
@@ -250,3 +251,18 @@ func (s *CephService) SetConfig(config map[string]string) {
 		s.config[key] = value
 	}
 }
+
+// SupportsFeature checks if the specified API feature of this Service instance if supported.
+func (s *CephService) SupportsFeature(ctx context.Context, feature string) (bool, error) {
+	server, err := s.m.Status(ctx)
+	if err != nil {
+		return false, fmt.Errorf("Failed to get MicroCeph server status while checking for features: %v", err)
+	}
+
+	if server.Extensions == nil {
+		logger.Warnf("MicroCeph server does not expose API extensions")
+		return false, nil
+	}
+
+	return server.Extensions.HasExtension(feature), nil
+}
diff --git a/service/microcloud.go b/service/microcloud.go
index 0ca6406a0..5f3222e7b 100644
--- a/service/microcloud.go
+++ b/service/microcloud.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/canonical/lxd/lxd/util"
 	"github.com/canonical/lxd/shared/api"
+	"github.com/canonical/lxd/shared/logger"
 	cephTypes "github.com/canonical/microceph/microceph/api/types"
 	microClient "github.com/canonical/microcluster/client"
 	"github.com/canonical/microcluster/config"
@@ -263,3 +264,18 @@ func (s *CloudService) SetConfig(config map[string]string) {
 		s.config[key] = value
 	}
 }
+
+// SupportsFeature checks if the specified API feature of this Service instance if supported.
+func (s *CloudService) SupportsFeature(ctx context.Context, feature string) (bool, error) {
+	server, err := s.client.Status(ctx)
+	if err != nil {
+		return false, fmt.Errorf("Failed to get MicroCloud server status while checking for features: %v", err)
+	}
+
+	if server.Extensions == nil {
+		logger.Warnf("MicroCloud server does not expose API extensions")
+		return false, nil
+	}
+
+	return server.Extensions.HasExtension(feature), nil
+}
diff --git a/service/microovn.go b/service/microovn.go
index 72954a60d..046400d86 100644
--- a/service/microovn.go
+++ b/service/microovn.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/canonical/lxd/lxd/util"
 	"github.com/canonical/lxd/shared"
+	"github.com/canonical/lxd/shared/logger"
 	"github.com/canonical/microcluster/client"
 	"github.com/canonical/microcluster/microcluster"
 
@@ -181,3 +182,18 @@ func (s *OVNService) SetConfig(config map[string]string) {
 		s.config[key] = value
 	}
 }
+
+// SupportsFeature checks if the specified API feature of this Service instance if supported.
+func (s *OVNService) SupportsFeature(ctx context.Context, feature string) (bool, error) {
+	server, err := s.m.Status(ctx)
+	if err != nil {
+		return false, fmt.Errorf("Failed to get MicroOVN server status while checking for features: %v", err)
+	}
+
+	if server.Extensions == nil {
+		logger.Warnf("MicroOVN server does not expose API extensions")
+		return false, nil
+	}
+
+	return server.Extensions.HasExtension(feature), nil
+}

From 935c2892e723a2c296a555767531c04037bd8000 Mon Sep 17 00:00:00 2001
From: Gabriel Mougard <gabriel.mougard@canonical.com>
Date: Wed, 31 Jul 2024 17:05:49 +0200
Subject: [PATCH 05/10] service: Add `OVNConfig` to the JoinConfig

Signed-off-by: Gabriel Mougard <gabriel.mougard@canonical.com>
---
 api/services.go       | 2 +-
 api/types/services.go | 1 +
 service/microcloud.go | 1 +
 service/microovn.go   | 2 +-
 4 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/api/services.go b/api/services.go
index 97a79a9f2..d7f34ce20 100644
--- a/api/services.go
+++ b/api/services.go
@@ -41,7 +41,7 @@ func servicesPut(state *state.State, r *http.Request) response.Response {
 	services := make([]types.ServiceType, len(req.Tokens))
 	for i, cfg := range req.Tokens {
 		services[i] = types.ServiceType(cfg.Service)
-		joinConfigs[cfg.Service] = service.JoinConfig{Token: cfg.JoinToken, LXDConfig: req.LXDConfig, CephConfig: req.CephConfig}
+		joinConfigs[cfg.Service] = service.JoinConfig{Token: cfg.JoinToken, LXDConfig: req.LXDConfig, CephConfig: req.CephConfig, OVNConfig: req.OVNConfig}
 	}
 
 	// Default to the first iface if none specified.
diff --git a/api/types/services.go b/api/types/services.go
index 2eec63841..b0b13d584 100644
--- a/api/types/services.go
+++ b/api/types/services.go
@@ -29,6 +29,7 @@ type ServicesPut struct {
 
 	LXDConfig  []api.ClusterMemberConfigKey `json:"lxd_config" yaml:"lxd_config"`
 	CephConfig []types.DisksPost            `json:"ceph_config" yaml:"ceph_config"`
+	OVNConfig  map[string]string            `json:"ovn_config" yaml:"ovn_config"`
 }
 
 // ServiceToken represents a join token for a service join request.
diff --git a/service/microcloud.go b/service/microcloud.go
index 5f3222e7b..3ffa895c5 100644
--- a/service/microcloud.go
+++ b/service/microcloud.go
@@ -36,6 +36,7 @@ type JoinConfig struct {
 	Token      string
 	LXDConfig  []api.ClusterMemberConfigKey
 	CephConfig []cephTypes.DisksPost
+	OVNConfig  map[string]string
 }
 
 // NewCloudService creates a new MicroCloud service with a client attached.
diff --git a/service/microovn.go b/service/microovn.go
index 046400d86..12b7affa3 100644
--- a/service/microovn.go
+++ b/service/microovn.go
@@ -114,7 +114,7 @@ func (s OVNService) DeleteToken(ctx context.Context, tokenName string, address s
 
 // Join joins a cluster with the given token.
 func (s OVNService) Join(ctx context.Context, joinConfig JoinConfig) error {
-	return s.m.JoinCluster(ctx, s.name, util.CanonicalNetworkAddress(s.address, s.port), joinConfig.Token, nil)
+	return s.m.JoinCluster(ctx, s.name, util.CanonicalNetworkAddress(s.address, s.port), joinConfig.Token, joinConfig.OVNConfig)
 }
 
 // RemoteClusterMembers returns a map of cluster member names and addresses from the MicroCloud at the given address, authenticated with the given secret.

From b9e97dd5006667d19b8222ebc7dbbac812c3cf57 Mon Sep 17 00:00:00 2001
From: Gabriel Mougard <gabriel.mougard@canonical.com>
Date: Thu, 1 Aug 2024 09:44:45 +0200
Subject: [PATCH 06/10] cmd/microcloud/ask: Interactively configure a custom
 underlay network

Signed-off-by: Gabriel Mougard <gabriel.mougard@canonical.com>
---
 cmd/microcloud/ask.go | 83 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)

diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go
index 40b4d81e2..2403b4165 100644
--- a/cmd/microcloud/ask.go
+++ b/cmd/microcloud/ask.go
@@ -1104,6 +1104,76 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
 		}
 	}
 
+	canOVNUnderlay := true
+	for peer, system := range c.systems {
+		if len(c.state[system.ServerInfo.Name].AvailableOVNInterfaces) == 0 {
+			fmt.Printf("Not enough interfaces available on %s to create an underlay network, skipping\n", peer)
+			canOVNUnderlay = false
+			break
+		}
+	}
+
+	var ovnUnderlaySelectedIPs map[string]string
+	ovnUnderlayData := [][]string{}
+	for peer, system := range c.systems {
+		// skip any systems that have already been clustered, but are available for other configuration.
+		state, ok := c.state[c.name]
+		if ok {
+			if state.ExistingServices[types.MicroOVN][peer] != "" {
+				continue
+			}
+		}
+
+		for _, net := range c.state[system.ServerInfo.Name].AvailableOVNInterfaces {
+			for _, addr := range net.Addresses {
+				ovnUnderlayData = append(ovnUnderlayData, []string{peer, net.Network.Name, net.Network.Type, addr})
+			}
+		}
+	}
+
+	if len(ovnUnderlayData) != 0 && canOVNUnderlay {
+		wantsDedicatedUnderlay, err := c.asker.AskBool("Configure dedicated underlay networking? (yes/no) [default=no]: ", "no")
+		if err != nil {
+			return err
+		}
+
+		if wantsDedicatedUnderlay {
+			header = []string{"LOCATION", "IFACE", "TYPE", "IP ADDRESS (CIDR)"}
+			fmt.Println("Select exactly one network interface from each cluster member:")
+
+			table = NewSelectableTable(header, ovnUnderlayData)
+			ovnUnderlaySelectedIPs = map[string]string{}
+			err = c.askRetry("Retry selecting underlay network interfaces?", func() error {
+				err = table.Render(table.rows)
+				if err != nil {
+					return err
+				}
+
+				answers, err := table.GetSelections()
+				if err != nil {
+					return err
+				}
+
+				ovnUnderlaySelectedIPs = map[string]string{}
+				for _, answer := range answers {
+					target := table.SelectionValue(answer, "LOCATION")
+					ipAddr := table.SelectionValue(answer, "IP ADDRESS (CIDR)")
+
+					if ovnUnderlaySelectedIPs[target] != "" {
+						return fmt.Errorf("Failed to configure OVN underlay traffic: Selected more than one interface for target %q", target)
+					}
+
+					ovnUnderlaySelectedIPs[target] = ipAddr
+				}
+
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	for peer, system := range c.systems {
 		if !askSystems[peer] {
 			continue
@@ -1133,6 +1203,19 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
 			system.Networks = append(system.Networks, finalConfigs...)
 		}
 
+		if ovnUnderlaySelectedIPs != nil {
+			ovnUnderlayIpAddr, ok := ovnUnderlaySelectedIPs[peer]
+			if ok {
+				ip, _, err := net.ParseCIDR(ovnUnderlayIpAddr)
+				if err != nil {
+					return err
+				}
+
+				fmt.Printf("Using %q for OVN underlay traffic on %q\n", ip.String(), peer)
+				system.OVNGeneveAddr = ip.String()
+			}
+		}
+
 		c.systems[peer] = system
 	}
 

From f3f2c9830542abe20f0f479350f6b5f158b364fa Mon Sep 17 00:00:00 2001
From: Gabriel Mougard <gabriel.mougard@canonical.com>
Date: Thu, 1 Aug 2024 10:01:00 +0200
Subject: [PATCH 07/10] microcloud/preseed: Allow the creation of an underlay
 network in preseed mode

Signed-off-by: Gabriel Mougard <gabriel.mougard@canonical.com>
---
 cmd/microcloud/main_init_preseed.go | 73 +++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/cmd/microcloud/main_init_preseed.go b/cmd/microcloud/main_init_preseed.go
index 411bc04c7..909da02e2 100644
--- a/cmd/microcloud/main_init_preseed.go
+++ b/cmd/microcloud/main_init_preseed.go
@@ -38,6 +38,7 @@ type Preseed struct {
 type System struct {
 	Name            string      `yaml:"name"`
 	UplinkInterface string      `yaml:"ovn_uplink_interface"`
+	UnderlayIP      string      `yaml:"underlay_ip"`
 	Storage         InitStorage `yaml:"storage"`
 }
 
@@ -220,6 +221,7 @@ func (c *initConfig) RunPreseed(cmd *cobra.Command) error {
 // validate validates the unmarshaled preseed input.
 func (p *Preseed) validate(name string, bootstrap bool) error {
 	uplinkCount := 0
+	underlayCount := 0
 	directCephCount := 0
 	directLocalCount := 0
 	localInit := false
@@ -241,6 +243,15 @@ func (p *Preseed) validate(name string, bootstrap bool) error {
 			uplinkCount++
 		}
 
+		if system.UnderlayIP != "" {
+			_, _, err := net.ParseCIDR(system.UnderlayIP)
+			if err != nil {
+				return fmt.Errorf("Invalid underlay IP: %w", err)
+			}
+
+			underlayCount++
+		}
+
 		if len(system.Storage.Ceph) > 0 {
 			directCephCount++
 		}
@@ -270,6 +281,11 @@ func (p *Preseed) validate(name string, bootstrap bool) error {
 		return fmt.Errorf("Some systems are missing an uplink interface")
 	}
 
+	containsUnderlay := underlayCount > 0
+	if containsUnderlay && underlayCount < len(p.Systems) {
+		return fmt.Errorf("Some systems are missing an underlay interface")
+	}
+
 	containsLocalStorage = directLocalCount > 0
 	if containsLocalStorage && directLocalCount < len(p.Systems) && len(p.Storage.Local) == 0 {
 		return fmt.Errorf("Some systems are missing local storage disks")
@@ -484,10 +500,15 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste
 
 	lxd := s.Services[types.LXD].(*service.LXDService)
 	ifaceByPeer := map[string]string{}
+	ovnUnderlayNeeded := false
 	for _, cfg := range p.Systems {
 		if cfg.UplinkInterface != "" {
 			ifaceByPeer[cfg.Name] = cfg.UplinkInterface
 		}
+
+		if cfg.UnderlayIP != "" {
+			ovnUnderlayNeeded = true
+		}
 	}
 
 	localInfo, err := s.CollectSystemInformation(context.Background(), mdns.ServerInfo{Name: c.name, Address: c.address})
@@ -542,6 +563,58 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste
 
 			c.systems[peer] = system
 		}
+
+		// Check the preseed underlay network configuration against the available ips.
+		if ovnUnderlayNeeded {
+			canOVNUnderlay := true
+			for peer, system := range c.state {
+				if len(system.AvailableOVNInterfaces) == 0 {
+					fmt.Printf("Not enough interfaces available on %s to create an underlay network, skipping\n", peer)
+					canOVNUnderlay = false
+					break
+				}
+			}
+
+			if canOVNUnderlay {
+				// TODO: call `s.Services[types.MicroOVN].(*service.OVNService).SupportsFeature(context.Background(), "custom_encapsulation_ip")`
+				// when MicroCloud will be updated with microcluster/v2
+				underlays := make(map[string]string, len(p.Systems))
+				for _, sys := range p.Systems {
+					underlays[sys.Name] = sys.UnderlayIP
+				}
+
+				underlayCount := 0
+				for _, sys := range p.Systems {
+					for _, net := range c.state[sys.Name].AvailableOVNInterfaces {
+						if len(net.Addresses) != 0 {
+							for _, cidrAddr := range net.Addresses {
+								if underlays[sys.Name] == cidrAddr {
+									underlayCount = underlayCount + 1
+									goto out
+								}
+							}
+						}
+					}
+
+				out:
+				}
+
+				if underlayCount != len(p.Systems) {
+					return nil, fmt.Errorf("Failed to find all underlay IPs on the network")
+				}
+
+				// Apply the underlay IPs to the systems.
+				for peer, system := range c.systems {
+					ip, _, err := net.ParseCIDR(underlays[peer])
+					if err != nil {
+						return nil, fmt.Errorf("Failed to parse underlay IP: %w", err)
+					}
+
+					system.OVNGeneveAddr = ip.String()
+					c.systems[peer] = system
+				}
+			}
+		}
 	} else {
 		// Check if FAN networking is usable.
 		fanUsable, _, err := service.FanNetworkUsable()

From 0e153d1a6c3d5b0b57226962e5fbde1acf943b3e Mon Sep 17 00:00:00 2001
From: Gabriel Mougard <gabriel.mougard@canonical.com>
Date: Thu, 1 Aug 2024 10:17:56 +0200
Subject: [PATCH 08/10] microcloud/cmd/microcloud: apply config for when a
 custom encaps Geneve IP is needed

Signed-off-by: Gabriel Mougard <gabriel.mougard@canonical.com>
---
 cmd/microcloud/main_init.go | 17 +++++++++++++++++
 service/microovn.go         |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go
index 4c99520b4..25d8d1613 100644
--- a/cmd/microcloud/main_init.go
+++ b/cmd/microcloud/main_init.go
@@ -527,6 +527,12 @@ func (c *initConfig) addPeers(sh *service.Handler) (revert.Hook, error) {
 			LXDConfig:  info.JoinConfig,
 			CephConfig: info.MicroCephDisks,
 		}
+
+		if info.OVNGeneveAddr != "" {
+			p := joinConfig[peer]
+			p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveAddr}
+			joinConfig[peer] = p
+		}
 	}
 
 	clusterSize := map[types.ServiceType]int{}
@@ -785,6 +791,17 @@ func (c *initConfig) setupCluster(s *service.Handler) error {
 			}
 		}
 
+		if s.Type() == types.MicroOVN {
+			microOvnBootstrapConf := make(map[string]string)
+			if bootstrapSystem.OVNGeneveAddr != "" {
+				microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveAddr
+			}
+
+			if len(microOvnBootstrapConf) > 0 {
+				s.SetConfig(microOvnBootstrapConf)
+			}
+		}
+
 		// set a 2 minute timeout to bootstrap a service in case the node is slow.
 		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 		defer cancel()
diff --git a/service/microovn.go b/service/microovn.go
index 12b7affa3..2c7210e78 100644
--- a/service/microovn.go
+++ b/service/microovn.go
@@ -59,7 +59,7 @@ func (s OVNService) Client() (*client.Client, error) {
 
 // Bootstrap bootstraps the MicroOVN daemon on the default port.
 func (s OVNService) Bootstrap(ctx context.Context) error {
-	err := s.m.NewCluster(ctx, s.name, util.CanonicalNetworkAddress(s.address, s.port), nil)
+	err := s.m.NewCluster(ctx, s.name, util.CanonicalNetworkAddress(s.address, s.port), s.config)
 	if err != nil {
 		return err
 	}

From 468b4a4f941e60b80391fc417f11e9a4333c7cc0 Mon Sep 17 00:00:00 2001
From: Gabriel Mougard <gabriel.mougard@canonical.com>
Date: Thu, 1 Aug 2024 10:19:36 +0200
Subject: [PATCH 09/10] doc: Add documentation for OVN underlay configuration

Signed-off-by: Gabriel Mougard <gabriel.mougard@canonical.com>
---
 doc/explanation/microcloud.md | 13 ++++++
 doc/how-to/index.md           |  1 +
 doc/how-to/initialise.md      |  7 +++
 doc/how-to/ovn_underlay.md    | 83 +++++++++++++++++++++++++++++++++++
 doc/tutorial/get_started.md   |  3 ++
 5 files changed, 107 insertions(+)
 create mode 100644 doc/how-to/ovn_underlay.md

diff --git a/doc/explanation/microcloud.md b/doc/explanation/microcloud.md
index cdb0fb03f..d9582014b 100644
--- a/doc/explanation/microcloud.md
+++ b/doc/explanation/microcloud.md
@@ -61,6 +61,19 @@ MicroCloud will still be usable, but you will see some limitations:
   As a result of this, network forwarding works at a basic level only, and external addresses must be forwarded to a specific cluster member and don't fail over.
 - There is no support for hardware acceleration, load balancers, or ACL functionality within the local network.
 
+(microcloud-networking-underlay)=
+
+Network interface to connect to an OVN underlay network
+:  While setting up the distributed networking with OVN, you can also decide to set up a dedicated underlay network for the OVN traffic.
+   A dedicated underlay network for OVN serves as the physical infrastructure over which the virtual networking (overlay network) is constructed. It can be beneficial to set up a dedicated underlay network for the following reasons:
+
+- A dedicated underlay network isolates the overlay network traffic from other types of traffic (such as management traffic).
+- By dedicating a physical network solely for the purpose of the virtual network overlay, network congestion can be reduced.
+- A dedicated underlay network can be tuned to provide predictable latency and bandwidth characteristics, which are crucial for the performance of latency-sensitive applications running over the OVN overlay.
+- A dedicated underlay can be designed to scale independently of other networks, which allows for more efficient scaling of the overlay network as the demand for virtual network resources increases.
+
+See {ref}`howto-ovn-underlay` for how to set up a dedicated OVN underlay network.
+
 ### Dedicated internal network for Ceph
 You can set up a dedicated network for Ceph to separate the internal Ceph traffic from the rest of the MicroCloud cluster traffic.
 
diff --git a/doc/how-to/index.md b/doc/how-to/index.md
index b6270e9f4..f091927f2 100644
--- a/doc/how-to/index.md
+++ b/doc/how-to/index.md
@@ -10,6 +10,7 @@ Install MicroCloud </how-to/install>
 Manage the snaps </how-to/snaps>
 Initialise MicroCloud </how-to/initialise>
 Configure Ceph networking </how-to/ceph_networking>
+Configure OVN underlay </how-to/ovn_underlay>
 Add a machine </how-to/add_machine>
 Remove a machine </how-to/remove_machine>
 Add a service </how-to/add_service>
diff --git a/doc/how-to/initialise.md b/doc/how-to/initialise.md
index 7db089f38..0e596d50a 100644
--- a/doc/how-to/initialise.md
+++ b/doc/how-to/initialise.md
@@ -90,6 +90,13 @@ Complete the following steps to initialise MicroCloud:
       You must select one network interface per machine.
    1. If you want to use IPv4, specify the IPv4 gateway on the uplink network (in CIDR notation) and the first and last IPv4 address in the range that you want to use with LXD.
    1. If you want to use IPv6, specify the IPv6 gateway on the uplink network (in CIDR notation).
+   1. If you chose to set up distributed networking, you can choose to setup an underlay network for the distributed networking:
+
+      If you choose ``yes``, configure the underlay network:
+
+      1. Select the network interfaces that you want to use (see {ref}`microcloud-networking-underlay`).
+
+         You must select one network interface with an IP address per machine.
 1. MicroCloud now starts to bootstrap the cluster.
    Monitor the output to see whether all steps complete successfully.
    See {ref}`bootstrapping-process` for more information.
diff --git a/doc/how-to/ovn_underlay.md b/doc/how-to/ovn_underlay.md
new file mode 100644
index 000000000..4a164324b
--- /dev/null
+++ b/doc/how-to/ovn_underlay.md
@@ -0,0 +1,83 @@
+(howto-ovn-underlay)=
+# How to configure an OVN underlay network
+
+When running {command}`microcloud init`, if you decided to setup distributed networking and if you have enough at least one unused network interface per cluster member with an IP address, you are asked if you want to configure an underlay network for OVN.
+Here is the question you will be asked:
+
+`Configure dedicated underlay networking? (yes/no) [default=no]: <answer>`
+
+You can choose to skip this question (just hit `Enter`). This will then use the internal MicroCloud network as an OVN 'underlay', which is the same as the OVN management network ('overlay' network).
+
+You could also choose to configure a dedicated underlay network for OVN with typing `yes`. A list of available network interfaces with an IP address will be displayed.
+You can then select one network interface per cluster member to be used as the interfaces for the underlay network of OVN.
+
+The following instructions build on the {ref}`get-started` tutorial and show how you can test setting up a MicroCloud with an OVN underlay network.
+
+1. Create the dedicated network for the OVN underlay:
+
+   1. First, create a dedicated network for the OVN cluster members to be used as an underlay. Let's call it `ovnbr0`:
+
+           lxc network create ovnbr0
+
+   1. Enter the following commands to find out the assigned IPv4 and IPv6 addresses for the networks and note them down:
+
+           lxc network get ovnbr0 ipv4.address
+           lxc network get ovnbr0 ipv6.address
+
+1. Create the network interfaces that will be used for the OVN underlay setup for each VM:
+
+   1. Add the network device for the `ovnbr0` network:
+
+           lxc config device add micro1 eth2 nic network=ovnbr0 name=eth2
+           lxc config device add micro2 eth2 nic network=ovnbr0 name=eth2
+           lxc config device add micro3 eth2 nic network=ovnbr0 name=eth2
+           lxc config device add micro4 eth2 nic network=ovnbr0 name=eth2
+
+1. Now, just like in the tutorial, start the VMs.
+1. On each VM, bring the network interfaces up and give them an IP address within their network subnet:
+
+   1. For the `ovnbr0` network, do the following for each VM::
+
+           # If the `ovnbr0` gateway address is `10.0.1.1/24` (subnet should be `10.0.1.0/24`)
+           ip link set enp7s0 up
+           # `X` should be a number between 2 and 254, different for each VM
+           ip addr add 10.0.1.X/24 dev enp7s0
+
+1. Now, you can start the MicroCloud initialisation process and provide the subnets you noted down when asked for the OVN underlay.
+1. We will use `ovnbr0` for the OVN underlay traffic. In a production setup, you'd choose the fast subnet for this traffic:
+
+           Configure dedicated underlay networking? (yes/no) [default=no]: yes
+           Select exactly one network interface from each cluster member:
+           Space to select; enter to confirm; type to filter results.
+           Up/down to move; right to select all; left to select none.
+                  +----------+--------+----------+-------------------------------------------+
+                  | LOCATION | IFACE  |   TYPE   |             IP ADDRESS (CIDR)             |
+                  +----------+--------+----------+-------------------------------------------+
+             [x]  | micro1   | enp7s0 | physical | 10.0.1.2/24                               |
+             [ ]  | micro1   | enp7s0 | physical | fd42:5782:5902:5b9e:216:3eff:fe01:67af/64 |
+             [x]  | micro3   | enp7s0 | physical | 10.0.1.4/24                               |
+             [ ]  | micro3   | enp7s0 | physical | fd42:5782:5902:5b9e:216:3eff:fe36:d29c/64 |
+           > [x]  | micro2   | enp7s0 | physical | 10.0.1.3/24                               |
+             [ ]  | micro2   | enp7s0 | physical | fd42:5782:5902:5b9e:216:3eff:fedb:f04e/64 |
+                  +----------+--------+----------+-------------------------------------------+
+
+1. The MicroCloud initialisation process will now continue as usual and the OVN cluster will be configured with the underlay network you provided.
+1. You can now inspect the OVN underlay setup:
+
+   1. Inspect the OVN southbound encapsulation parameters:
+
+     ```{terminal}
+       :input: microovn.ovn-sbctl --columns=ip,type find Encap type=geneve
+       :user: root
+       :host: micro1
+       :scroll:
+
+       ip                  : "10.77.55.2"
+       type                : geneve
+
+       ip                  : "10.77.55.4"
+       type                : geneve
+
+       ip                  : "10.77.55.3"
+       type                : geneve
+     ```
\ No newline at end of file
diff --git a/doc/tutorial/get_started.md b/doc/tutorial/get_started.md
index a16342ab0..dd7af0cee 100644
--- a/doc/tutorial/get_started.md
+++ b/doc/tutorial/get_started.md
@@ -287,6 +287,8 @@ Complete the following steps:
       As we're setting up four machines only, the range must contain a minimum of four addresses, but setting up a bigger range is more fail-safe.
       For example, if your IPv4 gateway is `192.0.2.1/24`, the last address could be `192.0.2.254`.
    1. Specify the IPv6 address that you noted down for your `microbr0` network as the IPv6 gateway.
+   1. Leave the question empty for the DNS addresses for the distributed network.
+   1. Leave the question empty for configuring an underlay network for OVN.
 
 MicroCloud will now initialise the cluster.
 See {ref}`explanation-initialisation` for more information.
@@ -415,6 +417,7 @@ Specify the first IPv4 address in the range to use on the uplink network: 192.0.
 Specify the last IPv4 address in the range to use on the uplink network: 192.0.2.254
 Specify the IPv6 gateway (CIDR) on the uplink network (empty to skip IPv6): 2001:db8:d:200::1/64
 Specify the DNS addresses (comma-separated IPv4 / IPv6 addresses) for the distributed network (default: 192.0.2.1,2001:db8:d:200::1):
+Configure dedicated underlay networking? (yes/no) [default=no]:
 
 Initializing a new cluster
  Local MicroCloud is ready

From ecdfb61d82037af4d2dc82fbd8dd03f6b34fe875 Mon Sep 17 00:00:00 2001
From: Gabriel Mougard <gabriel.mougard@canonical.com>
Date: Thu, 1 Aug 2024 10:26:43 +0200
Subject: [PATCH 10/10] tests: Add OVN underlay tests

Signed-off-by: Gabriel Mougard <gabriel.mougard@canonical.com>
---
 test/includes/microcloud.sh |  37 +++++++++++-
 test/suites/add.sh          |   3 +
 test/suites/basic.sh        | 109 ++++++++++++++++++++++++++++++++++++
 3 files changed, 146 insertions(+), 3 deletions(-)

diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh
index f4a804e6e..38e396cb3 100644
--- a/test/includes/microcloud.sh
+++ b/test/includes/microcloud.sh
@@ -2,10 +2,10 @@
 
 # unset_interactive_vars: Unsets all variables related to the test console.
 unset_interactive_vars() {
-  unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \
+  unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS PEERS_FILTER REUSE_EXISTING REUSE_EXISTING_COUNT \
     SETUP_ZFS ZFS_FILTER ZFS_WIPE \
     SETUP_CEPH CEPH_MISSING_DISKS CEPH_FILTER CEPH_WIPE CEPH_ENCRYPT SETUP_CEPHFS CEPH_CLUSTER_NETWORK \
-    PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \
+    PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_UNDERLAY_NETWORK OVN_UNDERLAY_FILTER OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \
     REPLACE_PROFILE CEPH_RETRY_HA MULTI_NODE
 }
 
@@ -27,6 +27,7 @@ microcloud_interactive() {
   LIMIT_SUBNET=${LIMIT_SUBNET:-}                 # (yes/no) input for limiting lookup of systems to the above subnet.
   SKIP_SERVICE=${SKIP_SERVICE:-}                 # (yes/no) input to skip any missing services. Should be unset if all services are installed.
   EXPECT_PEERS=${EXPECT_PEERS:-}                 # wait for this number of systems to be available to join the cluster.
+  PEERS_FILTER=${PEERS_FILTER:-}                 # filter string for the particular peer to init/add
   REUSE_EXISTING=${REUSE_EXISTING:-}              # (yes/no) incorporate an existing clustered service.
   REUSE_EXISTING_COUNT=${REUSE_EXISTING_COUNT:-0} # (number) number of existing clusters to incorporate.
   SETUP_ZFS=${SETUP_ZFS:-}                       # (yes/no) input for initiating ZFS storage pool setup.
@@ -48,6 +49,8 @@ microcloud_interactive() {
   IPV4_START=${IPV4_START:-}                      # OVN ipv4 range start.
   IPV4_END=${IPV4_END:-}                          # OVN ipv4 range end.
   DNS_ADDRESSES=${DNS_ADDRESSES:-}                # OVN custom DNS addresses.
+  OVN_UNDERLAY_NETWORK=${OVN_UNDERLAY_NETWORK:-}  # (yes/no) set up a custom OVN underlay network.
+  OVN_UNDERLAY_FILTER=${OVN_UNDERLAY_FILTER:-}    # filter string for OVN underlay interfaces.
   IPV6_SUBNET=${IPV6_SUBNET:-}                    # OVN ipv6 range.
   REPLACE_PROFILE="${REPLACE_PROFILE:-}"          # Replace default profile config and devices.
 
@@ -67,6 +70,7 @@ $(true)
 ${LIMIT_SUBNET}                                             # limit lookup subnet (yes/no)
 $([ "${SKIP_SERVICE}" = "yes" ] && printf "%s" "${SKIP_SERVICE}")  # skip MicroOVN/MicroCeph (yes/no)
 expect ${EXPECT_PEERS}                                      # wait until the systems show up
+${PEERS_FILTER}                                             # filter discovered peers
 select-all                                                  # select all the systems
 ---
 $(true)                                                 # workaround for set -e
@@ -136,6 +140,17 @@ ${IPV6_SUBNET}
 ${DNS_ADDRESSES}
 $(true)                                                 # workaround for set -e
 "
+
+  if [ -n "${OVN_UNDERLAY_NETWORK}" ]; then
+    setup="${setup}
+${OVN_UNDERLAY_NETWORK}
+$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf "wait 300ms")
+${OVN_UNDERLAY_FILTER}
+$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf "select-all")
+$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf -- "---")
+$(true)                                                 # workaround for set -e
+"
+  fi
 fi
 
 if [ -n "${REPLACE_PROFILE}" ] ; then
@@ -297,13 +312,29 @@ validate_ceph_encrypt() {
 # validate_system_microovn: Ensures the node with the given name has correctly set up MicroOVN with the given resources.
 validate_system_microovn() {
     name=${1}
+    shift 1
 
     echo "==> ${name} Validating MicroOVN"
 
     lxc remote switch local
     lxc exec "${name}" -- microovn cluster list | grep -q "${name}"
-}
 
+    ovn_underlay_subnet_prefix=""
+    if [ $# -gt 0 ] && echo "${1}" | grep -Pq '^([0-9]{1,3}\.){2}[0-9]{1,3}$'; then
+      ovn_underlay_subnet_prefix="${1}"
+      shift 1
+    fi
+
+    # Instances are named micro01, micro02, etc.
+    # We need to extract the instance number to check the OVN Geneve tunnel IP.
+    instance_idx=$(echo "${name}" | grep -oE '[0-9]+$')
+    underlay_ip_idx=$((instance_idx + 1))
+    if [ -n "${ovn_underlay_subnet_prefix}" ]; then
+      lxc exec "${name}" -- sh -ceu "
+        microovn.ovn-sbctl --columns=ip,type find Encap type=geneve | grep -q ${ovn_underlay_subnet_prefix}.${underlay_ip_idx}
+      " > /dev/null
+    fi
+}
 # validate_system_lxd_zfs: Ensures the node with the given name has the given disk set up for ZFS storage.
 validate_system_lxd_zfs() {
   name=${1}
diff --git a/test/suites/add.sh b/test/suites/add.sh
index 9e57cecd6..2ad5d299d 100644
--- a/test/suites/add.sh
+++ b/test/suites/add.sh
@@ -138,6 +138,7 @@ test_add_interactive() {
   export IPV4_END="10.1.123.254"
   export IPV6_SUBNET="fd42:1:1234:1234::1/64"
   export DNS_ADDRESSES="10.1.123.1,fd42:1:1234:1234::1"
+  export OVN_UNDERLAY_NETWORK="no"
   microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"
   lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
 
@@ -159,6 +160,7 @@ test_add_interactive() {
   export CEPH_ENCRYPT="no"
   export SETUP_OVN="yes"
   export OVN_FILTER="enp6s0"
+  export OVN_UNDERLAY_NETWORK="no"
   microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out"
   lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
 
@@ -237,6 +239,7 @@ test_add_interactive() {
   export IPV6_SUBNET="fd42:1:1234:1234::1/64"
   export DNS_ADDRESSES="10.1.123.1,fd42:1:1234:1234::1"
   export REPLACE_PROFILE="yes"
+  export OVN_UNDERLAY_NETWORK="no"
 
   lxc exec micro04 -- snap enable microcloud
   lxc exec micro04 -- snap start microcloud
diff --git a/test/suites/basic.sh b/test/suites/basic.sh
index 55fa8d512..7da15316b 100644
--- a/test/suites/basic.sh
+++ b/test/suites/basic.sh
@@ -137,6 +137,7 @@ test_interactive() {
   export IPV6_SUBNET="fd42:1:1234:1234::1/64"
   export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
   export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}"
+  export OVN_UNDERLAY_NETWORK="no"
   microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"
 
   lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
@@ -207,6 +208,7 @@ test_interactive() {
   export IPV4_SUBNET="10.1.123.1/24"
   export IPV4_START="10.1.123.100"
   export IPV4_END="10.1.123.254"
+  export OVN_UNDERLAY_NETWORK="no"
   microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"
 
   lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
@@ -215,6 +217,108 @@ test_interactive() {
     validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2
     validate_system_microovn "${m}"
   done
+
+  reset_systems 3 3 3
+
+  ceph_cluster_subnet_prefix="10.2.123"
+  ceph_cluster_subnet_iface="enp7s0"
+
+  for n in $(seq 2 4); do
+    cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24"
+    lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}"
+  done
+
+  ovn_underlay_subnet_prefix="10.3.123"
+  ovn_underlay_subnet_iface="enp8s0"
+
+  for n in $(seq 2 4); do
+    ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24"
+    lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up"
+  done
+
+  echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, OVN management network and OVN underlay network"
+  export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24"
+  export OVN_UNDERLAY_NETWORK="yes"
+  export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}"
+  microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"
+
+  lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
+  for m in micro01 micro02 micro03 ; do
+    validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}"
+    validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2
+    validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}"
+  done
+
+  reset_systems 3 3 2
+  echo "Add a MicroCloud node on a cluster of 2 nodes with MicroOVN not initialized"
+
+  ovn_underlay_subnet_prefix="10.2.123"
+  ovn_underlay_subnet_iface="enp7s0"
+
+  for n in $(seq 2 4); do
+    ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24"
+    lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up"
+  done
+
+  lxc exec micro03 -- snap disable microcloud || true
+  for m in micro01 micro02 ; do
+    lxc exec "${m}" -- snap disable microovn
+  done
+
+  unset_interactive_vars
+  microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)"
+  export MULTI_NODE="yes"
+  export LOOKUP_IFACE="enp5s0"
+  export LIMIT_SUBNET="yes"
+  export EXPECT_PEERS=1
+  export SKIP_SERVICE="yes"
+  export SETUP_ZFS="yes"
+  export ZFS_FILTER="lxd_disk1"
+  export ZFS_WIPE="yes"
+  export SETUP_CEPH="no"
+  export SETUP_OVN="no"
+
+  # Run a 2 nodes MicroCloud without MicroOVN first.
+  microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"
+
+  lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
+  for m in micro01 micro02 ; do
+    lxc exec "${m}" -- snap enable microovn
+  done
+
+  lxc exec micro03 -- microovn cluster bootstrap
+  lxc exec micro03 -- snap enable microcloud
+  lxc exec micro03 -- snap start microcloud
+
+  unset_interactive_vars
+  export LIMIT_SUBNET="yes"
+  export EXPECT_PEERS=2
+  export PEERS_FILTER="micro03"
+  export REUSE_EXISTING_COUNT=1
+  export REUSE_EXISTING="add"
+  export SETUP_ZFS="yes"
+  export ZFS_FILTER="lxd_disk1"
+  export ZFS_WIPE="yes"
+  export SETUP_CEPH="no"
+  export SETUP_OVN="yes"
+  export OVN_FILTER="enp6s0"
+  export OVN_UNDERLAY_NETWORK="yes"
+  export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}"
+  export IPV4_SUBNET="10.1.123.1/24"
+  export IPV4_START="10.1.123.100"
+  export IPV4_END="10.1.123.254"
+  export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
+  export IPV6_SUBNET="fd42:1:1234:1234::1/64"
+  export REPLACE_PROFILE="yes"
+  microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out"
+
+  for m in micro01 micro02 micro03 ; do
+    validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}"
+  done
+
+  for m in micro01 micro02 ; do
+    validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}"
+  done
 }
 
 test_instances_config() {
@@ -666,6 +770,7 @@ _test_case() {
     export LOOKUP_IFACE="enp5s0" # filter string for the lookup interface table.
     export LIMIT_SUBNET="yes" # (yes/no) input for limiting lookup of systems to the above subnet.
     export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}"
+    export OVN_UNDERLAY_NETWORK="no"
 
     export EXPECT_PEERS="$((num_systems - 1))"
 
@@ -1082,6 +1187,7 @@ test_reuse_cluster() {
   export IPV4_END="10.1.123.254"
   export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
   export IPV6_SUBNET="fd42:1:1234:1234::1/64"
+  export OVN_UNDERLAY_NETWORK="no"
 
   reset_systems 3 3 3
   echo "Create a MicroCloud that re-uses an existing service"
@@ -1210,6 +1316,7 @@ test_remove_cluster_member() {
   export IPV4_END="10.1.123.254"
   export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
   export IPV6_SUBNET="fd42:1:1234:1234::1/64"
+  export OVN_UNDERLAY_NETWORK="no"
 
   reset_systems 3 3 3
   echo "Fail to remove member from MicroCeph and LXD until OSDs are removed"
@@ -1358,6 +1465,7 @@ test_add_services() {
   export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
   export IPV6_SUBNET="fd42:1:1234:1234::1/64"
   export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24"
+  export OVN_UNDERLAY_NETWORK="no"
 
   reset_systems 3 3 3
   set_cluster_subnet 3  "${ceph_cluster_subnet_iface}" "${ceph_cluster_subnet_prefix}"
@@ -1495,6 +1603,7 @@ test_non_ha() {
   export CEPH_ENCRYPT="no"
   export CEPH_RETRY_HA="no"
   export SETUP_OVN="yes"
+  export OVN_UNDERLAY_NETWORK="no"
   export OVN_FILTER="enp6s0"
   export IPV4_SUBNET="10.1.123.1/24"
   export IPV4_START="10.1.123.100"