diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index a7c2131d..b56ee7db 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -625,32 +625,62 @@ func (c *CmdControl) askRemotePool(systems map[string]InitSystem, autoSetup bool return nil } +func (c *CmdControl) askProceedIfNoOverlayNetwork() error { + proceedWithNoOverlayNetworking, err := c.asker.AskBool("FAN networking is not usable. Do you want to proceed with setting up an inoperable cluster? (yes/no) [default=no]: ", "no") + if err != nil { + return err + } + + if proceedWithNoOverlayNetworking { + return nil + } + + return fmt.Errorf("cluster bootstrapping aborted due to lack of usable networking") +} + func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSystem, microCloudInternalSubnet *net.IPNet, autoSetup bool) error { _, bootstrap := systems[sh.Name] lxd := sh.Services[types.LXD].(*service.LXDService) - for peer, system := range systems { - if bootstrap { - system.TargetNetworks = []api.NetworksPost{lxd.DefaultPendingFanNetwork()} - if peer == sh.Name { - network, err := lxd.DefaultFanNetwork() - if err != nil { - return err - } - system.Networks = []api.NetworksPost{network} + // Check if FAN networking is usable. + fanUsable, _, err := lxd.FanNetworkUsable() + if err != nil { + return err + } + + if fanUsable { + for peer, system := range systems { + if bootstrap { + system.TargetNetworks = []api.NetworksPost{lxd.DefaultPendingFanNetwork()} + if peer == sh.Name { + network, err := lxd.DefaultFanNetwork() + if err != nil { + return err + } + + system.Networks = []api.NetworksPost{network} + } } - } - systems[peer] = system + systems[peer] = system + } } // Automatic setup gets a basic fan setup. if autoSetup { + if !fanUsable { + return c.askProceedIfNoOverlayNetwork() + } + return nil } // Environments without OVN get a basic fan setup. if sh.Services[types.MicroOVN] == nil { + if !fanUsable { + return c.askProceedIfNoOverlayNetwork() + } + return nil } @@ -769,6 +799,10 @@ func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSyst if !canOVN { fmt.Println("No dedicated uplink interfaces detected, skipping distributed networking") + if !fanUsable { + return c.askProceedIfNoOverlayNetwork() + } + return nil } diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index 07420703..756947ac 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -200,14 +200,13 @@ func lookupPeers(s *service.Handler, autoSetup bool, iface *net.Interface, subne }() } - var timeAfter <-chan time.Time + timeoutDuration := time.Minute if autoSetup { - timeAfter = time.After(5 * time.Second) + timeoutDuration = 5 * time.Second } - if len(expectedSystems) > 0 { - timeAfter = time.After(1 * time.Minute) - } + ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) + defer cancel() expectedSystemsMap := make(map[string]bool, len(expectedSystems)) for _, system := range expectedSystems { @@ -219,7 +218,7 @@ func lookupPeers(s *service.Handler, autoSetup bool, iface *net.Interface, subne done := false for !done { select { - case <-timeAfter: + case <-ctx.Done(): done = true case err := <-selectionCh: if err != nil { @@ -235,7 +234,7 @@ func lookupPeers(s *service.Handler, autoSetup bool, iface *net.Interface, subne break } - peers, err := mdns.LookupPeers(context.Background(), iface, mdns.Version, s.Name) + peers, err := mdns.LookupPeers(ctx, iface, mdns.Version, s.Name) if err != nil { return err } diff --git a/cmd/microcloud/main_init_preseed.go b/cmd/microcloud/main_init_preseed.go index 15ddac7e..9e6c396a 100644 --- a/cmd/microcloud/main_init_preseed.go +++ b/cmd/microcloud/main_init_preseed.go @@ -498,8 +498,14 @@ func (p *Preseed) Parse(s *service.Handler, bootstrap bool) (map[string]InitSyst systems[peer] = system } + // Check if FAN networking is usable. + fanUsable, _, err := lxd.FanNetworkUsable() + if err != nil { + return nil, err + } + // Setup FAN network if OVN not available. - if len(ifaceByPeer) == 0 { + if len(ifaceByPeer) == 0 && fanUsable { for peer, system := range systems { if bootstrap { system.TargetNetworks = append(system.TargetNetworks, lxd.DefaultPendingFanNetwork()) diff --git a/mdns/lookup.go b/mdns/lookup.go index a39160de..329fb2d8 100644 --- a/mdns/lookup.go +++ b/mdns/lookup.go @@ -148,7 +148,26 @@ func Lookup(ctx context.Context, iface *net.Interface, service string, size int) params.Interface = iface params.Entries = entriesCh params.Timeout = 100 * time.Millisecond - err := mdns.Query(params) + ipv4Supported, ipv6Supported, err := checkIPStatus(iface.Name) + if err != nil { + return nil, fmt.Errorf("Failed to check IP status: %w", err) + } + + if !ipv4Supported { + logger.Info("IPv4 is not supported on this system network interface: disabling IPv4 mDNS", logger.Ctx{"iface": iface.Name}) + params.DisableIPv4 = true + } + + if !ipv6Supported { + logger.Info("IPv6 is not supported on this system network interface: disabling IPv6 mDNS", logger.Ctx{"iface": iface.Name}) + params.DisableIPv6 = true + } + + if params.DisableIPv4 && params.DisableIPv6 { + return nil, fmt.Errorf("No supported IP versions on the network interface %q", iface.Name) + } + + err = mdns.Query(params) if err != nil { return nil, fmt.Errorf("Failed lookup: %w", err) } diff --git a/mdns/mdns.go b/mdns/mdns.go index b7bc4285..23387d44 100644 --- a/mdns/mdns.go +++ b/mdns/mdns.go @@ -48,3 +48,39 @@ func dnsTXTSlice(list []byte) []string { return parts } + +// checkIPStatus checks if the interface is up, has multicast, and supports IPv4/IPv6. +func checkIPStatus(iface string) (ipv4OK bool, ipv6OK bool, err error) { + netInterface, err := net.InterfaceByName(iface) + if err != nil { + return false, false, err + } + + if netInterface.Flags&net.FlagUp != net.FlagUp { + return false, false, nil + } + + if netInterface.Flags&net.FlagMulticast != net.FlagMulticast { + return false, false, nil + } + + addrs, err := netInterface.Addrs() + if err != nil { + return false, false, err + } + + for _, addr := range addrs { + ipNet, ok := addr.(*net.IPNet) + if !ok { + continue + } + + if ipNet.IP.To4() != nil { + ipv4OK = true + } else if ipNet.IP.To16() != nil { + ipv6OK = true + } + } + + return ipv4OK, ipv6OK, nil +} diff --git a/service/lxd.go b/service/lxd.go index 009c256b..ac01f9be 100644 --- a/service/lxd.go +++ b/service/lxd.go @@ -1,12 +1,10 @@ package service import ( - "bufio" "context" "fmt" "net" "net/url" - "os" "strings" "time" @@ -637,33 +635,14 @@ func (s *LXDService) waitReady(ctx context.Context, c lxd.InstanceServer, timeou } // defaultGatewaySubnetV4 returns subnet of default gateway interface. -func defaultGatewaySubnetV4() (*net.IPNet, string, error) { - file, err := os.Open("/proc/net/route") +func (s LXDService) defaultGatewaySubnetV4() (*net.IPNet, string, error) { + available, ifaceName, err := s.FanNetworkUsable() if err != nil { return nil, "", err } - defer func() { _ = file.Close() }() - - ifaceName := "" - - scanner := bufio.NewReader(file) - for { - line, _, err := scanner.ReadLine() - if err != nil { - break - } - - fields := strings.Fields(string(line)) - - if fields[1] == "00000000" && fields[7] == "00000000" { - ifaceName = fields[0] - break - } - } - - if ifaceName == "" { - return nil, "", fmt.Errorf("No default gateway for IPv4") + if !available { + return nil, "", fmt.Errorf("No default IPv4 gateway available") } iface, err := net.InterfaceByName(ifaceName) diff --git a/service/lxd_config.go b/service/lxd_config.go index 47a25461..9bed202b 100644 --- a/service/lxd_config.go +++ b/service/lxd_config.go @@ -1,9 +1,12 @@ package service import ( + "bufio" "fmt" "net" + "os" "strconv" + "strings" "github.com/canonical/lxd/shared/api" ) @@ -32,10 +35,42 @@ func (s LXDService) DefaultPendingFanNetwork() api.NetworksPost { return api.NetworksPost{Name: "lxdfan0", Type: "bridge"} } +// FanNetworkUsable checks if the current host is capable of using a Fan network. +// It actually checks if there is a default IPv4 gateway available. +func (s LXDService) FanNetworkUsable() (available bool, ifaceName string, err error) { + file, err := os.Open("/proc/net/route") + if err != nil { + return false, "", err + } + + defer func() { _ = file.Close() }() + + scanner := bufio.NewReader(file) + for { + line, _, err := scanner.ReadLine() + if err != nil { + break + } + + fields := strings.Fields(string(line)) + + if fields[1] == "00000000" && fields[7] == "00000000" { + ifaceName = fields[0] + break + } + } + + if ifaceName == "" { + return false, "", nil // There is no default gateway for IPv4 + } + + return true, ifaceName, nil +} + // DefaultFanNetwork returns the default Ubuntu Fan network configuration when // creating the finalized network. func (s LXDService) DefaultFanNetwork() (api.NetworksPost, error) { - underlay, _, err := defaultGatewaySubnetV4() + underlay, _, err := s.defaultGatewaySubnetV4() if err != nil { return api.NetworksPost{}, fmt.Errorf("Could not determine Fan overlay subnet: %w", err) } diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh index 7011bdf4..1faa2abd 100644 --- a/test/includes/microcloud.sh +++ b/test/includes/microcloud.sh @@ -5,7 +5,7 @@ unset_interactive_vars() { unset LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \ SETUP_ZFS ZFS_FILTER ZFS_WIPE \ SETUP_CEPH CEPH_WARNING CEPH_FILTER CEPH_WIPE SETUP_CEPHFS CEPH_CLUSTER_NETWORK IGNORE_CEPH_NETWORKING \ - SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET + PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET } # microcloud_interactive: outputs text that can be passed to `TEST_CONSOLE=1 microcloud init` @@ -19,24 +19,25 @@ microcloud_interactive() { EXPECT_PEERS=${EXPECT_PEERS:-} # wait for this number of systems to be available to join the cluster. REUSE_EXISTING=${REUSE_EXISTING:-} # (yes/no) incorporate an existing clustered service. REUSE_EXISTING_COUNT=${REUSE_EXISTING_COUNT:-0} # (number) number of existing clusters to incorporate. - SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup. - ZFS_FILTER=${ZFS_FILTER:-} # filter string for ZFS disks. - ZFS_WIPE=${ZFS_WIPE:-} # (yes/no) to wipe all disks. - SETUP_CEPH=${SETUP_CEPH:-} # (yes/no) input for initiating CEPH storage pool setup. - SETUP_CEPHFS=${SETUP_CEPHFS:-} # (yes/no) input for initialising CephFS storage pool setup. - CEPH_WARNING=${CEPH_WARNING:-} # (yes/no) input for warning about eligible disk detection. - CEPH_FILTER=${CEPH_FILTER:-} # filter string for CEPH disks. - CEPH_WIPE=${CEPH_WIPE:-} # (yes/no) to wipe all disks. + SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup. + ZFS_FILTER=${ZFS_FILTER:-} # filter string for ZFS disks. + ZFS_WIPE=${ZFS_WIPE:-} # (yes/no) to wipe all disks. + SETUP_CEPH=${SETUP_CEPH:-} # (yes/no) input for initiating CEPH storage pool setup. + SETUP_CEPHFS=${SETUP_CEPHFS:-} # (yes/no) input for initialising CephFS storage pool setup. + CEPH_WARNING=${CEPH_WARNING:-} # (yes/no) input for warning about eligible disk detection. + CEPH_FILTER=${CEPH_FILTER:-} # filter string for CEPH disks. + CEPH_WIPE=${CEPH_WIPE:-} # (yes/no) to wipe all disks. CEPH_CLUSTER_NETWORK=${CEPH_CLUSTER_NETWORK:-} # (default: MicroCloud internal subnet or Ceph public network if specified previously) input for setting up a cluster network. IGNORE_CEPH_NETWORKING=${IGNORE_CEPH_NETWORKING:-} # (yes/no) input for ignoring Ceph network setup. Set it to `yes` during `microcloud add` . - SETUP_OVN=${SETUP_OVN:-} # (yes/no) input for initiating OVN network setup. - OVN_WARNING=${OVN_WARNING:-} # (yes/no) input for warning about eligible interface detection. - OVN_FILTER=${OVN_FILTER:-} # filter string for OVN interfaces. - IPV4_SUBNET=${IPV4_SUBNET:-} # OVN ipv4 gateway subnet. - IPV4_START=${IPV4_START:-} # OVN ipv4 range start. - IPV4_END=${IPV4_END:-} # OVN ipv4 range end. - DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses. - IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range. + PROCEED_WITH_NO_OVERLAY_NETWORKING=${PROCEED_WITH_NO_OVERLAY_NETWORKING:-} # (yes/no) input for proceeding without overlay networking. + SETUP_OVN=${SETUP_OVN:-} # (yes/no) input for initiating OVN network setup. + OVN_WARNING=${OVN_WARNING:-} # (yes/no) input for warning about eligible interface detection. + OVN_FILTER=${OVN_FILTER:-} # filter string for OVN interfaces. + IPV4_SUBNET=${IPV4_SUBNET:-} # OVN ipv4 gateway subnet. + IPV4_START=${IPV4_START:-} # OVN ipv4 range start. + IPV4_END=${IPV4_END:-} # OVN ipv4 range end. + DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses. + IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range. setup=" ${LOOKUP_IFACE} # filter the lookup interface @@ -88,6 +89,13 @@ $(true) # workaround for set -e " fi +if [ -n "${PROCEED_WITH_NO_OVERLAY_NETWORKING}" ]; then + setup="${setup} +${PROCEED_WITH_NO_OVERLAY_NETWORKING} # agree to proceed without overlay networking (neither FAN nor OVN networking) (yes/no) +$(true) # workaround for set -e +" +fi + if [ -z "${IGNORE_CEPH_NETWORKING}" ]; then if [ -n "${CEPH_CLUSTER_NETWORK}" ]; then setup="${setup} diff --git a/test/suites/basic.sh b/test/suites/basic.sh index 5d636f0c..edd42aad 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -47,6 +47,46 @@ test_interactive() { validate_system_lxd "${m}" 3 disk1 done + # Reset the systems with just LXD and no IPv6 support. + reset_systems 3 3 1 + + for m in micro01 micro02 micro03 ; do + lxc exec "${m}" -- echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6 + lxc exec "${m}" -- snap disable microceph || true + lxc exec "${m}" -- snap disable microovn || true + lxc exec "${m}" -- snap restart microcloud + done + + echo "Creating a MicroCloud with ZFS storage and no IPv6 support" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + for m in micro01 micro02 micro03 ; do + validate_system_lxd "${m}" 3 disk1 + done + + # Reset the systems with just LXD and no IPv4 support. + gw_net_addr=$(lxc network get lxdbr0 ipv4.address) + lxc network set lxdbr0 ipv4.address none + reset_systems 3 3 1 + + for m in micro01 micro02 micro03 ; do + lxc exec "${m}" -- snap disable microceph || true + lxc exec "${m}" -- snap disable microovn || true + lxc exec "${m}" -- snap restart microcloud + done + + export PROCEED_WITH_NO_OVERLAY_NETWORKING="no" # This will avoid to setup the cluster if no overlay networking is available. + echo "Creating a MicroCloud with ZFS storage and no IPv4 support" + ! microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init 2> err" || false + + # Ensure we error out due to a lack of usable overlay networking. + lxc exec micro01 -- cat err | grep "cluster bootstrapping aborted due to lack of usable networking" -q + + # Set the IPv4 address back to the original value. + lxc network set lxdbr0 ipv4.address "${gw_net_addr}" + unset PROCEED_WITH_NO_OVERLAY_NETWORKING + # Reset the systems and install microceph. reset_systems 3 3 1