diff --git a/api/services.go b/api/services.go index 97a79a9f..d7f34ce2 100644 --- a/api/services.go +++ b/api/services.go @@ -41,7 +41,7 @@ func servicesPut(state *state.State, r *http.Request) response.Response { services := make([]types.ServiceType, len(req.Tokens)) for i, cfg := range req.Tokens { services[i] = types.ServiceType(cfg.Service) - joinConfigs[cfg.Service] = service.JoinConfig{Token: cfg.JoinToken, LXDConfig: req.LXDConfig, CephConfig: req.CephConfig} + joinConfigs[cfg.Service] = service.JoinConfig{Token: cfg.JoinToken, LXDConfig: req.LXDConfig, CephConfig: req.CephConfig, OVNConfig: req.OVNConfig} } // Default to the first iface if none specified. diff --git a/api/types/services.go b/api/types/services.go index 2eec6384..b0b13d58 100644 --- a/api/types/services.go +++ b/api/types/services.go @@ -29,6 +29,7 @@ type ServicesPut struct { LXDConfig []api.ClusterMemberConfigKey `json:"lxd_config" yaml:"lxd_config"` CephConfig []types.DisksPost `json:"ceph_config" yaml:"ceph_config"` + OVNConfig map[string]string `json:"ovn_config" yaml:"ovn_config"` } // ServiceToken represents a join token for a service join request. diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index 87441de8..2403b416 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -481,7 +481,7 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { return nil } -func validateCephInterfacesForSubnet(lxdService *service.LXDService, systems map[string]InitSystem, availableCephNetworkInterfaces map[string]map[string]service.CephDedicatedInterface, askedCephSubnet string) error { +func validateCephInterfacesForSubnet(lxdService *service.LXDService, systems map[string]InitSystem, availableCephNetworkInterfaces map[string]map[string]service.DedicatedInterface, askedCephSubnet string) error { validatedCephInterfacesData, err := lxdService.ValidateCephInterfaces(askedCephSubnet, availableCephNetworkInterfaces) if err != nil { return err @@ -1104,6 +1104,76 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { } } + canOVNUnderlay := true + for peer, system := range c.systems { + if len(c.state[system.ServerInfo.Name].AvailableOVNInterfaces) == 0 { + fmt.Printf("Not enough interfaces available on %s to create an underlay network, skipping\n", peer) + canOVNUnderlay = false + break + } + } + + var ovnUnderlaySelectedIPs map[string]string + ovnUnderlayData := [][]string{} + for peer, system := range c.systems { + // skip any systems that have already been clustered, but are available for other configuration. + state, ok := c.state[c.name] + if ok { + if state.ExistingServices[types.MicroOVN][peer] != "" { + continue + } + } + + for _, net := range c.state[system.ServerInfo.Name].AvailableOVNInterfaces { + for _, addr := range net.Addresses { + ovnUnderlayData = append(ovnUnderlayData, []string{peer, net.Network.Name, net.Network.Type, addr}) + } + } + } + + if len(ovnUnderlayData) != 0 && canOVNUnderlay { + wantsDedicatedUnderlay, err := c.asker.AskBool("Configure dedicated underlay networking? (yes/no) [default=no]: ", "no") + if err != nil { + return err + } + + if wantsDedicatedUnderlay { + header = []string{"LOCATION", "IFACE", "TYPE", "IP ADDRESS (CIDR)"} + fmt.Println("Select exactly one network interface from each cluster member:") + + table = NewSelectableTable(header, ovnUnderlayData) + ovnUnderlaySelectedIPs = map[string]string{} + err = c.askRetry("Retry selecting underlay network interfaces?", func() error { + err = table.Render(table.rows) + if err != nil { + return err + } + + answers, err := table.GetSelections() + if err != nil { + return err + } + + ovnUnderlaySelectedIPs = map[string]string{} + for _, answer := range answers { + target := table.SelectionValue(answer, "LOCATION") + ipAddr := table.SelectionValue(answer, "IP ADDRESS (CIDR)") + + if ovnUnderlaySelectedIPs[target] != "" { + return fmt.Errorf("Failed to configure OVN underlay traffic: Selected more than one interface for target %q", target) + } + + ovnUnderlaySelectedIPs[target] = ipAddr + } + + return nil + }) + if err != nil { + return err + } + } + } + for peer, system := range c.systems { if !askSystems[peer] { continue @@ -1133,6 +1203,19 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { system.Networks = append(system.Networks, finalConfigs...) } + if ovnUnderlaySelectedIPs != nil { + ovnUnderlayIpAddr, ok := ovnUnderlaySelectedIPs[peer] + if ok { + ip, _, err := net.ParseCIDR(ovnUnderlayIpAddr) + if err != nil { + return err + } + + fmt.Printf("Using %q for OVN underlay traffic on %q\n", ip.String(), peer) + system.OVNGeneveAddr = ip.String() + } + } + c.systems[peer] = system } @@ -1220,7 +1303,7 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error { return nil } - availableCephNetworkInterfaces := map[string]map[string]service.CephDedicatedInterface{} + availableCephNetworkInterfaces := map[string]map[string]service.DedicatedInterface{} for name, state := range c.state { if len(state.AvailableCephInterfaces) == 0 { fmt.Printf("No network interfaces found with IPs on %q to set a dedicated Ceph network, skipping Ceph network setup\n", name) @@ -1228,7 +1311,7 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error { return nil } - ifaces := make(map[string]service.CephDedicatedInterface, len(state.AvailableCephInterfaces)) + ifaces := make(map[string]service.DedicatedInterface, len(state.AvailableCephInterfaces)) for name, iface := range state.AvailableCephInterfaces { ifaces[name] = iface } diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index 8dab2fe0..25d8d161 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -53,6 +53,9 @@ type InitSystem struct { TargetStoragePools []lxdAPI.StoragePoolsPost // Networks is the cluster-wide network configuration. Networks []lxdAPI.NetworksPost + // OVNGeneveAddr represents an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system. + // If left empty, the system will choose to route the Geneve traffic through the management network. + OVNGeneveAddr string // StoragePools is the cluster-wide storage pool configuration. StoragePools []lxdAPI.StoragePoolsPost // StorageVolumes is the cluster-wide storage volume configuration. @@ -524,6 +527,12 @@ func (c *initConfig) addPeers(sh *service.Handler) (revert.Hook, error) { LXDConfig: info.JoinConfig, CephConfig: info.MicroCephDisks, } + + if info.OVNGeneveAddr != "" { + p := joinConfig[peer] + p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveAddr} + joinConfig[peer] = p + } } clusterSize := map[types.ServiceType]int{} @@ -782,6 +791,17 @@ func (c *initConfig) setupCluster(s *service.Handler) error { } } + if s.Type() == types.MicroOVN { + microOvnBootstrapConf := make(map[string]string) + if bootstrapSystem.OVNGeneveAddr != "" { + microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveAddr + } + + if len(microOvnBootstrapConf) > 0 { + s.SetConfig(microOvnBootstrapConf) + } + } + // set a 2 minute timeout to bootstrap a service in case the node is slow. ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() diff --git a/cmd/microcloud/main_init_preseed.go b/cmd/microcloud/main_init_preseed.go index 61b99859..909da02e 100644 --- a/cmd/microcloud/main_init_preseed.go +++ b/cmd/microcloud/main_init_preseed.go @@ -38,6 +38,7 @@ type Preseed struct { type System struct { Name string `yaml:"name"` UplinkInterface string `yaml:"ovn_uplink_interface"` + UnderlayIP string `yaml:"underlay_ip"` Storage InitStorage `yaml:"storage"` } @@ -220,6 +221,7 @@ func (c *initConfig) RunPreseed(cmd *cobra.Command) error { // validate validates the unmarshaled preseed input. func (p *Preseed) validate(name string, bootstrap bool) error { uplinkCount := 0 + underlayCount := 0 directCephCount := 0 directLocalCount := 0 localInit := false @@ -241,6 +243,15 @@ func (p *Preseed) validate(name string, bootstrap bool) error { uplinkCount++ } + if system.UnderlayIP != "" { + _, _, err := net.ParseCIDR(system.UnderlayIP) + if err != nil { + return fmt.Errorf("Invalid underlay IP: %w", err) + } + + underlayCount++ + } + if len(system.Storage.Ceph) > 0 { directCephCount++ } @@ -270,6 +281,11 @@ func (p *Preseed) validate(name string, bootstrap bool) error { return fmt.Errorf("Some systems are missing an uplink interface") } + containsUnderlay := underlayCount > 0 + if containsUnderlay && underlayCount < len(p.Systems) { + return fmt.Errorf("Some systems are missing an underlay interface") + } + containsLocalStorage = directLocalCount > 0 if containsLocalStorage && directLocalCount < len(p.Systems) && len(p.Storage.Local) == 0 { return fmt.Errorf("Some systems are missing local storage disks") @@ -484,10 +500,15 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste lxd := s.Services[types.LXD].(*service.LXDService) ifaceByPeer := map[string]string{} + ovnUnderlayNeeded := false for _, cfg := range p.Systems { if cfg.UplinkInterface != "" { ifaceByPeer[cfg.Name] = cfg.UplinkInterface } + + if cfg.UnderlayIP != "" { + ovnUnderlayNeeded = true + } } localInfo, err := s.CollectSystemInformation(context.Background(), mdns.ServerInfo{Name: c.name, Address: c.address}) @@ -498,7 +519,7 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste // If an uplink interface was explicitly chosen, we will try to set up an OVN network. explicitOVN := len(ifaceByPeer) > 0 - cephInterfaces := map[string]map[string]service.CephDedicatedInterface{} + cephInterfaces := map[string]map[string]service.DedicatedInterface{} for _, system := range c.systems { uplinkIfaces, cephIfaces, _, err := lxd.GetNetworkInterfaces(context.Background(), system.ServerInfo.Name, system.ServerInfo.Address, system.ServerInfo.AuthSecret) if err != nil { @@ -517,7 +538,7 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste for ifaceName, iface := range cephIfaces { if cephInterfaces[system.ServerInfo.Name] == nil { - cephInterfaces[system.ServerInfo.Name] = map[string]service.CephDedicatedInterface{} + cephInterfaces[system.ServerInfo.Name] = map[string]service.DedicatedInterface{} } cephInterfaces[system.ServerInfo.Name][ifaceName] = iface @@ -542,6 +563,58 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig) (map[string]InitSyste c.systems[peer] = system } + + // Check the preseed underlay network configuration against the available ips. + if ovnUnderlayNeeded { + canOVNUnderlay := true + for peer, system := range c.state { + if len(system.AvailableOVNInterfaces) == 0 { + fmt.Printf("Not enough interfaces available on %s to create an underlay network, skipping\n", peer) + canOVNUnderlay = false + break + } + } + + if canOVNUnderlay { + // TODO: call `s.Services[types.MicroOVN].(*service.OVNService).SupportsFeature(context.Background(), "custom_encapsulation_ip")` + // when MicroCloud will be updated with microcluster/v2 + underlays := make(map[string]string, len(p.Systems)) + for _, sys := range p.Systems { + underlays[sys.Name] = sys.UnderlayIP + } + + underlayCount := 0 + for _, sys := range p.Systems { + for _, net := range c.state[sys.Name].AvailableOVNInterfaces { + if len(net.Addresses) != 0 { + for _, cidrAddr := range net.Addresses { + if underlays[sys.Name] == cidrAddr { + underlayCount = underlayCount + 1 + goto out + } + } + } + } + + out: + } + + if underlayCount != len(p.Systems) { + return nil, fmt.Errorf("Failed to find all underlay IPs on the network") + } + + // Apply the underlay IPs to the systems. + for peer, system := range c.systems { + ip, _, err := net.ParseCIDR(underlays[peer]) + if err != nil { + return nil, fmt.Errorf("Failed to parse underlay IP: %w", err) + } + + system.OVNGeneveAddr = ip.String() + c.systems[peer] = system + } + } + } } else { // Check if FAN networking is usable. fanUsable, _, err := service.FanNetworkUsable() diff --git a/doc/explanation/microcloud.md b/doc/explanation/microcloud.md index cdb0fb03..d9582014 100644 --- a/doc/explanation/microcloud.md +++ b/doc/explanation/microcloud.md @@ -61,6 +61,19 @@ MicroCloud will still be usable, but you will see some limitations: As a result of this, network forwarding works at a basic level only, and external addresses must be forwarded to a specific cluster member and don't fail over. - There is no support for hardware acceleration, load balancers, or ACL functionality within the local network. +(microcloud-networking-underlay)= + +Network interface to connect to an OVN underlay network +: While setting up the distributed networking with OVN, you can also decide to set up a dedicated underlay network for the OVN traffic. + A dedicated underlay network for OVN serves as the physical infrastructure over which the virtual networking (overlay network) is constructed. It can be beneficial to set up a dedicated underlay network for the following reasons: + +- A dedicated underlay network isolates the overlay network traffic from other types of traffic (such as management traffic). +- By dedicating a physical network solely for the purpose of the virtual network overlay, network congestion can be reduced. +- A dedicated underlay network can be tuned to provide predictable latency and bandwidth characteristics, which are crucial for the performance of latency-sensitive applications running over the OVN overlay. +- A dedicated underlay can be designed to scale independently of other networks, which allows for more efficient scaling of the overlay network as the demand for virtual network resources increases. + +See {ref}`howto-ovn-underlay` for how to set up a dedicated OVN underlay network. + ### Dedicated internal network for Ceph You can set up a dedicated network for Ceph to separate the internal Ceph traffic from the rest of the MicroCloud cluster traffic. diff --git a/doc/how-to/index.md b/doc/how-to/index.md index b6270e9f..f091927f 100644 --- a/doc/how-to/index.md +++ b/doc/how-to/index.md @@ -10,6 +10,7 @@ Install MicroCloud Manage the snaps Initialise MicroCloud Configure Ceph networking +Configure OVN underlay Add a machine Remove a machine Add a service diff --git a/doc/how-to/initialise.md b/doc/how-to/initialise.md index 7db089f3..0e596d50 100644 --- a/doc/how-to/initialise.md +++ b/doc/how-to/initialise.md @@ -90,6 +90,13 @@ Complete the following steps to initialise MicroCloud: You must select one network interface per machine. 1. If you want to use IPv4, specify the IPv4 gateway on the uplink network (in CIDR notation) and the first and last IPv4 address in the range that you want to use with LXD. 1. If you want to use IPv6, specify the IPv6 gateway on the uplink network (in CIDR notation). + 1. If you chose to set up distributed networking, you can choose to setup an underlay network for the distributed networking: + + If you choose ``yes``, configure the underlay network: + + 1. Select the network interfaces that you want to use (see {ref}`microcloud-networking-underlay`). + + You must select one network interface with an IP address per machine. 1. MicroCloud now starts to bootstrap the cluster. Monitor the output to see whether all steps complete successfully. See {ref}`bootstrapping-process` for more information. diff --git a/doc/how-to/ovn_underlay.md b/doc/how-to/ovn_underlay.md new file mode 100644 index 00000000..4a164324 --- /dev/null +++ b/doc/how-to/ovn_underlay.md @@ -0,0 +1,83 @@ +(howto-ovn-underlay)= +# How to configure an OVN underlay network + +When running {command}`microcloud init`, if you decided to setup distributed networking and if you have enough at least one unused network interface per cluster member with an IP address, you are asked if you want to configure an underlay network for OVN. +Here is the question you will be asked: + +`Configure dedicated underlay networking? (yes/no) [default=no]: ` + +You can choose to skip this question (just hit `Enter`). This will then use the internal MicroCloud network as an OVN 'underlay', which is the same as the OVN management network ('overlay' network). + +You could also choose to configure a dedicated underlay network for OVN with typing `yes`. A list of available network interfaces with an IP address will be displayed. +You can then select one network interface per cluster member to be used as the interfaces for the underlay network of OVN. + +The following instructions build on the {ref}`get-started` tutorial and show how you can test setting up a MicroCloud with an OVN underlay network. + +1. Create the dedicated network for the OVN underlay: + + 1. First, create a dedicated network for the OVN cluster members to be used as an underlay. Let's call it `ovnbr0`: + + lxc network create ovnbr0 + + 1. Enter the following commands to find out the assigned IPv4 and IPv6 addresses for the networks and note them down: + + lxc network get ovnbr0 ipv4.address + lxc network get ovnbr0 ipv6.address + +1. Create the network interfaces that will be used for the OVN underlay setup for each VM: + + 1. Add the network device for the `ovnbr0` network: + + lxc config device add micro1 eth2 nic network=ovnbr0 name=eth2 + lxc config device add micro2 eth2 nic network=ovnbr0 name=eth2 + lxc config device add micro3 eth2 nic network=ovnbr0 name=eth2 + lxc config device add micro4 eth2 nic network=ovnbr0 name=eth2 + +1. Now, just like in the tutorial, start the VMs. +1. On each VM, bring the network interfaces up and give them an IP address within their network subnet: + + 1. For the `ovnbr0` network, do the following for each VM:: + + # If the `ovnbr0` gateway address is `10.0.1.1/24` (subnet should be `10.0.1.0/24`) + ip link set enp7s0 up + # `X` should be a number between 2 and 254, different for each VM + ip addr add 10.0.1.X/24 dev enp7s0 + +1. Now, you can start the MicroCloud initialisation process and provide the subnets you noted down when asked for the OVN underlay. +1. We will use `ovnbr0` for the OVN underlay traffic. In a production setup, you'd choose the fast subnet for this traffic: + + Configure dedicated underlay networking? (yes/no) [default=no]: yes + Select exactly one network interface from each cluster member: + Space to select; enter to confirm; type to filter results. + Up/down to move; right to select all; left to select none. + +----------+--------+----------+-------------------------------------------+ + | LOCATION | IFACE | TYPE | IP ADDRESS (CIDR) | + +----------+--------+----------+-------------------------------------------+ + [x] | micro1 | enp7s0 | physical | 10.0.1.2/24 | + [ ] | micro1 | enp7s0 | physical | fd42:5782:5902:5b9e:216:3eff:fe01:67af/64 | + [x] | micro3 | enp7s0 | physical | 10.0.1.4/24 | + [ ] | micro3 | enp7s0 | physical | fd42:5782:5902:5b9e:216:3eff:fe36:d29c/64 | + > [x] | micro2 | enp7s0 | physical | 10.0.1.3/24 | + [ ] | micro2 | enp7s0 | physical | fd42:5782:5902:5b9e:216:3eff:fedb:f04e/64 | + +----------+--------+----------+-------------------------------------------+ + +1. The MicroCloud initialisation process will now continue as usual and the OVN cluster will be configured with the underlay network you provided. +1. You can now inspect the OVN underlay setup: + + 1. Inspect the OVN southbound encapsulation parameters: + + ```{terminal} + :input: microovn.ovn-sbctl --columns=ip,type find Encap type=geneve + :user: root + :host: micro1 + :scroll: + + ip : "10.77.55.2" + type : geneve + + ip : "10.77.55.4" + type : geneve + + ip : "10.77.55.3" + type : geneve + ``` \ No newline at end of file diff --git a/doc/tutorial/get_started.md b/doc/tutorial/get_started.md index a16342ab..dd7af0ce 100644 --- a/doc/tutorial/get_started.md +++ b/doc/tutorial/get_started.md @@ -287,6 +287,8 @@ Complete the following steps: As we're setting up four machines only, the range must contain a minimum of four addresses, but setting up a bigger range is more fail-safe. For example, if your IPv4 gateway is `192.0.2.1/24`, the last address could be `192.0.2.254`. 1. Specify the IPv6 address that you noted down for your `microbr0` network as the IPv6 gateway. + 1. Leave the question empty for the DNS addresses for the distributed network. + 1. Leave the question empty for configuring an underlay network for OVN. MicroCloud will now initialise the cluster. See {ref}`explanation-initialisation` for more information. @@ -415,6 +417,7 @@ Specify the first IPv4 address in the range to use on the uplink network: 192.0. Specify the last IPv4 address in the range to use on the uplink network: 192.0.2.254 Specify the IPv6 gateway (CIDR) on the uplink network (empty to skip IPv6): 2001:db8:d:200::1/64 Specify the DNS addresses (comma-separated IPv4 / IPv6 addresses) for the distributed network (default: 192.0.2.1,2001:db8:d:200::1): +Configure dedicated underlay networking? (yes/no) [default=no]: Initializing a new cluster Local MicroCloud is ready diff --git a/go.mod b/go.mod index c757dde7..8346faf9 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/canonical/lxd v0.0.0-20240709081608-9df996e36bac github.com/canonical/microceph/microceph v0.0.0-20240708123233-b17a96f27a3d github.com/canonical/microcluster v0.0.0-20240709163503-3d587817830c - github.com/canonical/microovn/microovn v0.0.0-20240703103351-546b832b3167 + github.com/canonical/microovn/microovn v0.0.0-20240710084353-a14a37d36234 github.com/creack/pty v1.1.21 github.com/gorilla/mux v1.8.1 github.com/hashicorp/mdns v1.0.5 diff --git a/go.sum b/go.sum index e82f1b56..532b4fb9 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/canonical/microceph/microceph v0.0.0-20240708123233-b17a96f27a3d h1:s github.com/canonical/microceph/microceph v0.0.0-20240708123233-b17a96f27a3d/go.mod h1:bfL5e3XQSSoWlwNDVTnBlu8+38kRIV7kMNvhioxFBQ4= github.com/canonical/microcluster v0.0.0-20240709163503-3d587817830c h1:Kjz8OZ2rFcpGfF9KYSJ6iAWraK131IQlO/13VB+saQ0= github.com/canonical/microcluster v0.0.0-20240709163503-3d587817830c/go.mod h1:ldDJ32PQhoc7sTSkQnUs2CQPnjUOkkXrzlbUaq9EIPI= -github.com/canonical/microovn/microovn v0.0.0-20240703103351-546b832b3167 h1:rEMSn67Nd68PEDAt0GE/tUEQpOGEwfuY0eyDR0TBMWo= -github.com/canonical/microovn/microovn v0.0.0-20240703103351-546b832b3167/go.mod h1:d0NHxbKZTvNjxAurMFNkAT/aCkBEg4owKB2HjNXZaMQ= +github.com/canonical/microovn/microovn v0.0.0-20240710084353-a14a37d36234 h1:jLP6bhGfzajUtNty6peVqHGrldRlDdgA2seeDIedCqI= +github.com/canonical/microovn/microovn v0.0.0-20240710084353-a14a37d36234/go.mod h1:SBLx+SJM/I9ZoQCipdOBLC94MqvO5DJdTLrm4sGeW9k= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= diff --git a/service/interface.go b/service/interface.go index cfa045ab..aa90d585 100644 --- a/service/interface.go +++ b/service/interface.go @@ -24,4 +24,5 @@ type Service interface { Address() string Port() int64 SetConfig(config map[string]string) + SupportsFeature(ctx context.Context, feature string) (bool, error) } diff --git a/service/lxd.go b/service/lxd.go index d4e858fa..ad8edbab 100644 --- a/service/lxd.go +++ b/service/lxd.go @@ -518,24 +518,26 @@ func defaultNetworkInterfacesFilter(client lxd.InstanceServer, network api.Netwo continue } - addresses = append(addresses, address.Address) + addresses = append(addresses, fmt.Sprintf("%s/%s", address.Address, address.Netmask)) } } return false, addresses } -// CephDedicatedInterface represents a dedicated interface for Ceph. -type CephDedicatedInterface struct { +// DedicatedInterface represents a dedicated interface for OVN. +type DedicatedInterface struct { Type string + Network api.Network Addresses []string } // GetNetworkInterfaces fetches the list of networks from LXD and returns the following: // - A map of uplink compatible networks keyed by interface name. // - A map of ceph compatible networks keyed by interface name. +// - A map of ovn compatible networks keyed by interface name. // - The list of all networks. -func (s LXDService) GetNetworkInterfaces(ctx context.Context, name string, address string, secret string) (map[string]api.Network, map[string]CephDedicatedInterface, []api.Network, error) { +func (s LXDService) GetNetworkInterfaces(ctx context.Context, name string, address string, secret string) (map[string]api.Network, map[string]DedicatedInterface, []api.Network, error) { var err error var client lxd.InstanceServer if name == s.Name() { @@ -554,7 +556,7 @@ func (s LXDService) GetNetworkInterfaces(ctx context.Context, name string, addre } uplinkInterfaces := map[string]api.Network{} - cephInterfaces := map[string]CephDedicatedInterface{} + dedicatedInterfaces := map[string]DedicatedInterface{} for _, network := range networks { filtered, addresses := defaultNetworkInterfacesFilter(client, network) if filtered { @@ -564,19 +566,20 @@ func (s LXDService) GetNetworkInterfaces(ctx context.Context, name string, addre if len(addresses) == 0 { uplinkInterfaces[network.Name] = network } else { - cephInterfaces[network.Name] = CephDedicatedInterface{ + dedicatedInterfaces[network.Name] = DedicatedInterface{ Type: network.Type, + Network: network, Addresses: addresses, } } } - return uplinkInterfaces, cephInterfaces, networks, nil + return uplinkInterfaces, dedicatedInterfaces, networks, nil } // ValidateCephInterfaces validates the given interfaces map against the given Ceph network subnet // and returns a map of peer name to interfaces that are in the subnet. -func (s *LXDService) ValidateCephInterfaces(cephNetworkSubnetStr string, peerInterfaces map[string]map[string]CephDedicatedInterface) (map[string][][]string, error) { +func (s *LXDService) ValidateCephInterfaces(cephNetworkSubnetStr string, peerInterfaces map[string]map[string]DedicatedInterface) (map[string][][]string, error) { _, subnet, err := net.ParseCIDR(cephNetworkSubnetStr) if err != nil { return nil, fmt.Errorf("Invalid CIDR subnet: %v", err) @@ -593,7 +596,11 @@ func (s *LXDService) ValidateCephInterfaces(cephNetworkSubnetStr string, peerInt for _, addr := range iface.Addresses { ip := net.ParseIP(addr) if ip == nil { - return nil, fmt.Errorf("Invalid IP address: %v", addr) + // Attempt to parse the IP address as a CIDR. + ip, _, err = net.ParseCIDR(addr) + if err != nil { + return nil, fmt.Errorf("Could not parse either IP nor CIDR notation for address %q: %v", addr, err) + } } if (subnet.IP.To4() != nil && ip.To4() != nil && subnet.Contains(ip)) || (subnet.IP.To16() != nil && ip.To16() != nil && subnet.Contains(ip)) { @@ -776,3 +783,22 @@ func (s LXDService) defaultGatewaySubnetV4() (*net.IPNet, string, error) { return subnet, ifaceName, nil } + +// SupportsFeature checks if the specified API feature of this Service instance if supported. +func (s LXDService) SupportsFeature(ctx context.Context, feature string) (bool, error) { + c, err := s.Client(ctx, "") + if err != nil { + return false, err + } + + server, _, err := c.GetServer() + if err != nil { + return false, err + } + + if server.APIExtensions == nil { + return false, fmt.Errorf("API extensions not available when checking for a LXD feature") + } + + return shared.ValueInSlice(feature, server.APIExtensions), nil +} diff --git a/service/microceph.go b/service/microceph.go index 0da37716..55502158 100644 --- a/service/microceph.go +++ b/service/microceph.go @@ -10,6 +10,7 @@ import ( "github.com/canonical/lxd/lxd/util" "github.com/canonical/lxd/shared" + "github.com/canonical/lxd/shared/logger" cephTypes "github.com/canonical/microceph/microceph/api/types" cephClient "github.com/canonical/microceph/microceph/client" "github.com/canonical/microcluster/client" @@ -250,3 +251,18 @@ func (s *CephService) SetConfig(config map[string]string) { s.config[key] = value } } + +// SupportsFeature checks if the specified API feature of this Service instance if supported. +func (s *CephService) SupportsFeature(ctx context.Context, feature string) (bool, error) { + server, err := s.m.Status(ctx) + if err != nil { + return false, fmt.Errorf("Failed to get MicroCeph server status while checking for features: %v", err) + } + + if server.Extensions == nil { + logger.Warnf("MicroCeph server does not expose API extensions") + return false, nil + } + + return server.Extensions.HasExtension(feature), nil +} diff --git a/service/microcloud.go b/service/microcloud.go index 0ca6406a..3ffa895c 100644 --- a/service/microcloud.go +++ b/service/microcloud.go @@ -8,6 +8,7 @@ import ( "github.com/canonical/lxd/lxd/util" "github.com/canonical/lxd/shared/api" + "github.com/canonical/lxd/shared/logger" cephTypes "github.com/canonical/microceph/microceph/api/types" microClient "github.com/canonical/microcluster/client" "github.com/canonical/microcluster/config" @@ -35,6 +36,7 @@ type JoinConfig struct { Token string LXDConfig []api.ClusterMemberConfigKey CephConfig []cephTypes.DisksPost + OVNConfig map[string]string } // NewCloudService creates a new MicroCloud service with a client attached. @@ -263,3 +265,18 @@ func (s *CloudService) SetConfig(config map[string]string) { s.config[key] = value } } + +// SupportsFeature checks if the specified API feature of this Service instance if supported. +func (s *CloudService) SupportsFeature(ctx context.Context, feature string) (bool, error) { + server, err := s.client.Status(ctx) + if err != nil { + return false, fmt.Errorf("Failed to get MicroCloud server status while checking for features: %v", err) + } + + if server.Extensions == nil { + logger.Warnf("MicroCloud server does not expose API extensions") + return false, nil + } + + return server.Extensions.HasExtension(feature), nil +} diff --git a/service/microovn.go b/service/microovn.go index 72954a60..2c7210e7 100644 --- a/service/microovn.go +++ b/service/microovn.go @@ -10,6 +10,7 @@ import ( "github.com/canonical/lxd/lxd/util" "github.com/canonical/lxd/shared" + "github.com/canonical/lxd/shared/logger" "github.com/canonical/microcluster/client" "github.com/canonical/microcluster/microcluster" @@ -58,7 +59,7 @@ func (s OVNService) Client() (*client.Client, error) { // Bootstrap bootstraps the MicroOVN daemon on the default port. func (s OVNService) Bootstrap(ctx context.Context) error { - err := s.m.NewCluster(ctx, s.name, util.CanonicalNetworkAddress(s.address, s.port), nil) + err := s.m.NewCluster(ctx, s.name, util.CanonicalNetworkAddress(s.address, s.port), s.config) if err != nil { return err } @@ -113,7 +114,7 @@ func (s OVNService) DeleteToken(ctx context.Context, tokenName string, address s // Join joins a cluster with the given token. func (s OVNService) Join(ctx context.Context, joinConfig JoinConfig) error { - return s.m.JoinCluster(ctx, s.name, util.CanonicalNetworkAddress(s.address, s.port), joinConfig.Token, nil) + return s.m.JoinCluster(ctx, s.name, util.CanonicalNetworkAddress(s.address, s.port), joinConfig.Token, joinConfig.OVNConfig) } // RemoteClusterMembers returns a map of cluster member names and addresses from the MicroCloud at the given address, authenticated with the given secret. @@ -181,3 +182,18 @@ func (s *OVNService) SetConfig(config map[string]string) { s.config[key] = value } } + +// SupportsFeature checks if the specified API feature of this Service instance if supported. +func (s *OVNService) SupportsFeature(ctx context.Context, feature string) (bool, error) { + server, err := s.m.Status(ctx) + if err != nil { + return false, fmt.Errorf("Failed to get MicroOVN server status while checking for features: %v", err) + } + + if server.Extensions == nil { + logger.Warnf("MicroOVN server does not expose API extensions") + return false, nil + } + + return server.Extensions.HasExtension(feature), nil +} diff --git a/service/system_information.go b/service/system_information.go index d18eb9d0..7c57dee8 100644 --- a/service/system_information.go +++ b/service/system_information.go @@ -33,7 +33,10 @@ type SystemInformation struct { AvailableUplinkInterfaces map[string]api.Network // AvailableCephInterfaces is the list of networks that can be used for the Ceph cluster network. - AvailableCephInterfaces map[string]CephDedicatedInterface + AvailableCephInterfaces map[string]DedicatedInterface + + // AvailableOVNInterfaces is the list of networks that can be used for an OVN underlay network. + AvailableOVNInterfaces map[string]DedicatedInterface // LXDLocalConfig is the local configuration of LXD on this system. LXDLocalConfig map[string]any @@ -78,7 +81,8 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mdn AuthSecret: connectInfo.AuthSecret, AvailableDisks: map[string]api.ResourcesStorageDisk{}, AvailableUplinkInterfaces: map[string]api.Network{}, - AvailableCephInterfaces: map[string]CephDedicatedInterface{}, + AvailableCephInterfaces: map[string]DedicatedInterface{}, + AvailableOVNInterfaces: map[string]DedicatedInterface{}, } var err error @@ -108,11 +112,15 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mdn } var allNets []api.Network - s.AvailableUplinkInterfaces, s.AvailableCephInterfaces, allNets, err = lxd.GetNetworkInterfaces(ctx, s.ClusterName, s.ClusterAddress, s.AuthSecret) + uplinkInterfaces, dedicatedInterfaces, allNets, err := lxd.GetNetworkInterfaces(ctx, s.ClusterName, s.ClusterAddress, s.AuthSecret) if err != nil { return nil, fmt.Errorf("Failed to get network interfaces on %q: %w", s.ClusterName, err) } + s.AvailableUplinkInterfaces = uplinkInterfaces + s.AvailableCephInterfaces = dedicatedInterfaces + s.AvailableOVNInterfaces = dedicatedInterfaces + for _, network := range allNets { if network.Name == DefaultFANNetwork { s.existingFanNetwork = &network diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh index f4a804e6..38e396cb 100644 --- a/test/includes/microcloud.sh +++ b/test/includes/microcloud.sh @@ -2,10 +2,10 @@ # unset_interactive_vars: Unsets all variables related to the test console. unset_interactive_vars() { - unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \ + unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS PEERS_FILTER REUSE_EXISTING REUSE_EXISTING_COUNT \ SETUP_ZFS ZFS_FILTER ZFS_WIPE \ SETUP_CEPH CEPH_MISSING_DISKS CEPH_FILTER CEPH_WIPE CEPH_ENCRYPT SETUP_CEPHFS CEPH_CLUSTER_NETWORK \ - PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \ + PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_UNDERLAY_NETWORK OVN_UNDERLAY_FILTER OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \ REPLACE_PROFILE CEPH_RETRY_HA MULTI_NODE } @@ -27,6 +27,7 @@ microcloud_interactive() { LIMIT_SUBNET=${LIMIT_SUBNET:-} # (yes/no) input for limiting lookup of systems to the above subnet. SKIP_SERVICE=${SKIP_SERVICE:-} # (yes/no) input to skip any missing services. Should be unset if all services are installed. EXPECT_PEERS=${EXPECT_PEERS:-} # wait for this number of systems to be available to join the cluster. + PEERS_FILTER=${PEERS_FILTER:-} # filter string for the particular peer to init/add REUSE_EXISTING=${REUSE_EXISTING:-} # (yes/no) incorporate an existing clustered service. REUSE_EXISTING_COUNT=${REUSE_EXISTING_COUNT:-0} # (number) number of existing clusters to incorporate. SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup. @@ -48,6 +49,8 @@ microcloud_interactive() { IPV4_START=${IPV4_START:-} # OVN ipv4 range start. IPV4_END=${IPV4_END:-} # OVN ipv4 range end. DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses. + OVN_UNDERLAY_NETWORK=${OVN_UNDERLAY_NETWORK:-} # (yes/no) set up a custom OVN underlay network. + OVN_UNDERLAY_FILTER=${OVN_UNDERLAY_FILTER:-} # filter string for OVN underlay interfaces. IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range. REPLACE_PROFILE="${REPLACE_PROFILE:-}" # Replace default profile config and devices. @@ -67,6 +70,7 @@ $(true) ${LIMIT_SUBNET} # limit lookup subnet (yes/no) $([ "${SKIP_SERVICE}" = "yes" ] && printf "%s" "${SKIP_SERVICE}") # skip MicroOVN/MicroCeph (yes/no) expect ${EXPECT_PEERS} # wait until the systems show up +${PEERS_FILTER} # filter discovered peers select-all # select all the systems --- $(true) # workaround for set -e @@ -136,6 +140,17 @@ ${IPV6_SUBNET} ${DNS_ADDRESSES} $(true) # workaround for set -e " + + if [ -n "${OVN_UNDERLAY_NETWORK}" ]; then + setup="${setup} +${OVN_UNDERLAY_NETWORK} +$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf "wait 300ms") +${OVN_UNDERLAY_FILTER} +$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf "select-all") +$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf -- "---") +$(true) # workaround for set -e +" + fi fi if [ -n "${REPLACE_PROFILE}" ] ; then @@ -297,13 +312,29 @@ validate_ceph_encrypt() { # validate_system_microovn: Ensures the node with the given name has correctly set up MicroOVN with the given resources. validate_system_microovn() { name=${1} + shift 1 echo "==> ${name} Validating MicroOVN" lxc remote switch local lxc exec "${name}" -- microovn cluster list | grep -q "${name}" -} + ovn_underlay_subnet_prefix="" + if [ $# -gt 0 ] && echo "${1}" | grep -Pq '^([0-9]{1,3}\.){2}[0-9]{1,3}$'; then + ovn_underlay_subnet_prefix="${1}" + shift 1 + fi + + # Instances are named micro01, micro02, etc. + # We need to extract the instance number to check the OVN Geneve tunnel IP. + instance_idx=$(echo "${name}" | grep -oE '[0-9]+$') + underlay_ip_idx=$((instance_idx + 1)) + if [ -n "${ovn_underlay_subnet_prefix}" ]; then + lxc exec "${name}" -- sh -ceu " + microovn.ovn-sbctl --columns=ip,type find Encap type=geneve | grep -q ${ovn_underlay_subnet_prefix}.${underlay_ip_idx} + " > /dev/null + fi +} # validate_system_lxd_zfs: Ensures the node with the given name has the given disk set up for ZFS storage. validate_system_lxd_zfs() { name=${1} diff --git a/test/suites/add.sh b/test/suites/add.sh index 9e57cecd..2ad5d299 100644 --- a/test/suites/add.sh +++ b/test/suites/add.sh @@ -138,6 +138,7 @@ test_add_interactive() { export IPV4_END="10.1.123.254" export IPV6_SUBNET="fd42:1:1234:1234::1/64" export DNS_ADDRESSES="10.1.123.1,fd42:1:1234:1234::1" + export OVN_UNDERLAY_NETWORK="no" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -159,6 +160,7 @@ test_add_interactive() { export CEPH_ENCRYPT="no" export SETUP_OVN="yes" export OVN_FILTER="enp6s0" + export OVN_UNDERLAY_NETWORK="no" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -237,6 +239,7 @@ test_add_interactive() { export IPV6_SUBNET="fd42:1:1234:1234::1/64" export DNS_ADDRESSES="10.1.123.1,fd42:1:1234:1234::1" export REPLACE_PROFILE="yes" + export OVN_UNDERLAY_NETWORK="no" lxc exec micro04 -- snap enable microcloud lxc exec micro04 -- snap start microcloud diff --git a/test/suites/basic.sh b/test/suites/basic.sh index 55fa8d51..7da15316 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -137,6 +137,7 @@ test_interactive() { export IPV6_SUBNET="fd42:1:1234:1234::1/64" export DNS_ADDRESSES="10.1.123.1,8.8.8.8" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" + export OVN_UNDERLAY_NETWORK="no" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -207,6 +208,7 @@ test_interactive() { export IPV4_SUBNET="10.1.123.1/24" export IPV4_START="10.1.123.100" export IPV4_END="10.1.123.254" + export OVN_UNDERLAY_NETWORK="no" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -215,6 +217,108 @@ test_interactive() { validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 validate_system_microovn "${m}" done + + reset_systems 3 3 3 + + ceph_cluster_subnet_prefix="10.2.123" + ceph_cluster_subnet_iface="enp7s0" + + for n in $(seq 2 4); do + cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" + lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" + done + + ovn_underlay_subnet_prefix="10.3.123" + ovn_underlay_subnet_iface="enp8s0" + + for n in $(seq 2 4); do + ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24" + lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up" + done + + echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, OVN management network and OVN underlay network" + export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" + export OVN_UNDERLAY_NETWORK="yes" + export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + for m in micro01 micro02 micro03 ; do + validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" + validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 + validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}" + done + + reset_systems 3 3 2 + echo "Add a MicroCloud node on a cluster of 2 nodes with MicroOVN not initialized" + + ovn_underlay_subnet_prefix="10.2.123" + ovn_underlay_subnet_iface="enp7s0" + + for n in $(seq 2 4); do + ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24" + lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up" + done + + lxc exec micro03 -- snap disable microcloud || true + for m in micro01 micro02 ; do + lxc exec "${m}" -- snap disable microovn + done + + unset_interactive_vars + microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + export MULTI_NODE="yes" + export LOOKUP_IFACE="enp5s0" + export LIMIT_SUBNET="yes" + export EXPECT_PEERS=1 + export SKIP_SERVICE="yes" + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="no" + export SETUP_OVN="no" + + # Run a 2 nodes MicroCloud without MicroOVN first. + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + for m in micro01 micro02 ; do + lxc exec "${m}" -- snap enable microovn + done + + lxc exec micro03 -- microovn cluster bootstrap + lxc exec micro03 -- snap enable microcloud + lxc exec micro03 -- snap start microcloud + + unset_interactive_vars + export LIMIT_SUBNET="yes" + export EXPECT_PEERS=2 + export PEERS_FILTER="micro03" + export REUSE_EXISTING_COUNT=1 + export REUSE_EXISTING="add" + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="no" + export SETUP_OVN="yes" + export OVN_FILTER="enp6s0" + export OVN_UNDERLAY_NETWORK="yes" + export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}" + export IPV4_SUBNET="10.1.123.1/24" + export IPV4_START="10.1.123.100" + export IPV4_END="10.1.123.254" + export DNS_ADDRESSES="10.1.123.1,8.8.8.8" + export IPV6_SUBNET="fd42:1:1234:1234::1/64" + export REPLACE_PROFILE="yes" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out" + + for m in micro01 micro02 micro03 ; do + validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" + done + + for m in micro01 micro02 ; do + validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}" + done } test_instances_config() { @@ -666,6 +770,7 @@ _test_case() { export LOOKUP_IFACE="enp5s0" # filter string for the lookup interface table. export LIMIT_SUBNET="yes" # (yes/no) input for limiting lookup of systems to the above subnet. export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" + export OVN_UNDERLAY_NETWORK="no" export EXPECT_PEERS="$((num_systems - 1))" @@ -1082,6 +1187,7 @@ test_reuse_cluster() { export IPV4_END="10.1.123.254" export DNS_ADDRESSES="10.1.123.1,8.8.8.8" export IPV6_SUBNET="fd42:1:1234:1234::1/64" + export OVN_UNDERLAY_NETWORK="no" reset_systems 3 3 3 echo "Create a MicroCloud that re-uses an existing service" @@ -1210,6 +1316,7 @@ test_remove_cluster_member() { export IPV4_END="10.1.123.254" export DNS_ADDRESSES="10.1.123.1,8.8.8.8" export IPV6_SUBNET="fd42:1:1234:1234::1/64" + export OVN_UNDERLAY_NETWORK="no" reset_systems 3 3 3 echo "Fail to remove member from MicroCeph and LXD until OSDs are removed" @@ -1358,6 +1465,7 @@ test_add_services() { export DNS_ADDRESSES="10.1.123.1,8.8.8.8" export IPV6_SUBNET="fd42:1:1234:1234::1/64" export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" + export OVN_UNDERLAY_NETWORK="no" reset_systems 3 3 3 set_cluster_subnet 3 "${ceph_cluster_subnet_iface}" "${ceph_cluster_subnet_prefix}" @@ -1495,6 +1603,7 @@ test_non_ha() { export CEPH_ENCRYPT="no" export CEPH_RETRY_HA="no" export SETUP_OVN="yes" + export OVN_UNDERLAY_NETWORK="no" export OVN_FILTER="enp6s0" export IPV4_SUBNET="10.1.123.1/24" export IPV4_START="10.1.123.100"