From 392567684a4b5135c2b26fb0320819e530a13f09 Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Tue, 12 Dec 2023 19:21:47 +0100 Subject: [PATCH 1/7] microcloud/cmd: fix timeout issue In case of mDNS discovery issue (no IPv6 support on a node for example), the timeout never seems to occur leading to wait forever. This should fix the issue Signed-off-by: Gabriel Mougard --- cmd/microcloud/main_init.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index 074207038..756947ac3 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -200,14 +200,13 @@ func lookupPeers(s *service.Handler, autoSetup bool, iface *net.Interface, subne }() } - var timeAfter <-chan time.Time + timeoutDuration := time.Minute if autoSetup { - timeAfter = time.After(5 * time.Second) + timeoutDuration = 5 * time.Second } - if len(expectedSystems) > 0 { - timeAfter = time.After(1 * time.Minute) - } + ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) + defer cancel() expectedSystemsMap := make(map[string]bool, len(expectedSystems)) for _, system := range expectedSystems { @@ -219,7 +218,7 @@ func lookupPeers(s *service.Handler, autoSetup bool, iface *net.Interface, subne done := false for !done { select { - case <-timeAfter: + case <-ctx.Done(): done = true case err := <-selectionCh: if err != nil { @@ -235,7 +234,7 @@ func lookupPeers(s *service.Handler, autoSetup bool, iface *net.Interface, subne break } - peers, err := mdns.LookupPeers(context.Background(), iface, mdns.Version, s.Name) + peers, err := mdns.LookupPeers(ctx, iface, mdns.Version, s.Name) if err != nil { return err } From 6ad0137702973f233097f3434653bee3050d589b Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Tue, 5 Dec 2023 14:52:27 +0100 Subject: [PATCH 2/7] mdns: add utility to check if host supports IPv6 Signed-off-by: Gabriel Mougard --- mdns/mdns.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/mdns/mdns.go b/mdns/mdns.go index b7bc42854..23387d44b 100644 --- a/mdns/mdns.go +++ b/mdns/mdns.go @@ -48,3 +48,39 @@ func dnsTXTSlice(list []byte) []string { return parts } + +// checkIPStatus checks if the interface is up, has multicast, and supports IPv4/IPv6. +func checkIPStatus(iface string) (ipv4OK bool, ipv6OK bool, err error) { + netInterface, err := net.InterfaceByName(iface) + if err != nil { + return false, false, err + } + + if netInterface.Flags&net.FlagUp != net.FlagUp { + return false, false, nil + } + + if netInterface.Flags&net.FlagMulticast != net.FlagMulticast { + return false, false, nil + } + + addrs, err := netInterface.Addrs() + if err != nil { + return false, false, err + } + + for _, addr := range addrs { + ipNet, ok := addr.(*net.IPNet) + if !ok { + continue + } + + if ipNet.IP.To4() != nil { + ipv4OK = true + } else if ipNet.IP.To16() != nil { + ipv6OK = true + } + } + + return ipv4OK, ipv6OK, nil +} From 52d5697b4fe609b0fb7703979deb8a21ff55875d Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Tue, 5 Dec 2023 14:53:20 +0100 Subject: [PATCH 3/7] mdns: if host does not support IPv6, disable mDNS IPv6 networking Signed-off-by: Gabriel Mougard --- mdns/lookup.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/mdns/lookup.go b/mdns/lookup.go index a39160ded..329fb2d83 100644 --- a/mdns/lookup.go +++ b/mdns/lookup.go @@ -148,7 +148,26 @@ func Lookup(ctx context.Context, iface *net.Interface, service string, size int) params.Interface = iface params.Entries = entriesCh params.Timeout = 100 * time.Millisecond - err := mdns.Query(params) + ipv4Supported, ipv6Supported, err := checkIPStatus(iface.Name) + if err != nil { + return nil, fmt.Errorf("Failed to check IP status: %w", err) + } + + if !ipv4Supported { + logger.Info("IPv4 is not supported on this system network interface: disabling IPv4 mDNS", logger.Ctx{"iface": iface.Name}) + params.DisableIPv4 = true + } + + if !ipv6Supported { + logger.Info("IPv6 is not supported on this system network interface: disabling IPv6 mDNS", logger.Ctx{"iface": iface.Name}) + params.DisableIPv6 = true + } + + if params.DisableIPv4 && params.DisableIPv6 { + return nil, fmt.Errorf("No supported IP versions on the network interface %q", iface.Name) + } + + err = mdns.Query(params) if err != nil { return nil, fmt.Errorf("Failed lookup: %w", err) } From 6d9c2b4ddc69596ca8dd37bae4394f9209fe3ba4 Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Fri, 7 Jun 2024 10:31:35 +0200 Subject: [PATCH 4/7] service/lxd: Introduce a FAN networking check function Signed-off-by: Gabriel Mougard --- service/lxd.go | 29 ++++------------------------- service/lxd_config.go | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/service/lxd.go b/service/lxd.go index d09ed65f6..7aa21c11a 100644 --- a/service/lxd.go +++ b/service/lxd.go @@ -1,13 +1,11 @@ package service import ( - "bufio" "context" "fmt" "net" "net/http" "net/url" - "os" "strings" "time" @@ -638,33 +636,14 @@ func (s *LXDService) waitReady(ctx context.Context, c lxd.InstanceServer, timeou } // defaultGatewaySubnetV4 returns subnet of default gateway interface. -func defaultGatewaySubnetV4() (*net.IPNet, string, error) { - file, err := os.Open("/proc/net/route") +func (s LXDService) defaultGatewaySubnetV4() (*net.IPNet, string, error) { + available, ifaceName, err := s.FanNetworkUsable() if err != nil { return nil, "", err } - defer func() { _ = file.Close() }() - - ifaceName := "" - - scanner := bufio.NewReader(file) - for { - line, _, err := scanner.ReadLine() - if err != nil { - break - } - - fields := strings.Fields(string(line)) - - if fields[1] == "00000000" && fields[7] == "00000000" { - ifaceName = fields[0] - break - } - } - - if ifaceName == "" { - return nil, "", fmt.Errorf("No default gateway for IPv4") + if !available { + return nil, "", fmt.Errorf("No default IPv4 gateway available") } iface, err := net.InterfaceByName(ifaceName) diff --git a/service/lxd_config.go b/service/lxd_config.go index ccd8b4c79..8525f9097 100644 --- a/service/lxd_config.go +++ b/service/lxd_config.go @@ -1,9 +1,12 @@ package service import ( + "bufio" "fmt" "net" + "os" "strconv" + "strings" "github.com/canonical/lxd/shared/api" ) @@ -23,10 +26,42 @@ func (s LXDService) DefaultPendingFanNetwork() api.NetworksPost { return api.NetworksPost{Name: "lxdfan0", Type: "bridge"} } +// FanNetworkUsable checks if the current host is capable of using a Fan network. +// It actually checks if there is a default IPv4 gateway available. +func (s LXDService) FanNetworkUsable() (available bool, ifaceName string, err error) { + file, err := os.Open("/proc/net/route") + if err != nil { + return false, "", err + } + + defer func() { _ = file.Close() }() + + scanner := bufio.NewReader(file) + for { + line, _, err := scanner.ReadLine() + if err != nil { + break + } + + fields := strings.Fields(string(line)) + + if fields[1] == "00000000" && fields[7] == "00000000" { + ifaceName = fields[0] + break + } + } + + if ifaceName == "" { + return false, "", nil // There is no default gateway for IPv4 + } + + return true, ifaceName, nil +} + // DefaultFanNetwork returns the default Ubuntu Fan network configuration when // creating the finalized network. func (s LXDService) DefaultFanNetwork() (api.NetworksPost, error) { - underlay, _, err := defaultGatewaySubnetV4() + underlay, _, err := s.defaultGatewaySubnetV4() if err != nil { return api.NetworksPost{}, fmt.Errorf("Could not determine Fan overlay subnet: %w", err) } From 1a7f0dbbce216c7fb9871e14c12f18fc6adf2da2 Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Fri, 7 Jun 2024 10:32:10 +0200 Subject: [PATCH 5/7] cdm/microcloud: Apply FAN network check during the setup Signed-off-by: Gabriel Mougard --- cmd/microcloud/ask.go | 43 +++++++++++++++++++++-------- cmd/microcloud/main_init_preseed.go | 8 +++++- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index a7c2131da..4ef23f05d 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -628,29 +628,46 @@ func (c *CmdControl) askRemotePool(systems map[string]InitSystem, autoSetup bool func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSystem, microCloudInternalSubnet *net.IPNet, autoSetup bool) error { _, bootstrap := systems[sh.Name] lxd := sh.Services[types.LXD].(*service.LXDService) - for peer, system := range systems { - if bootstrap { - system.TargetNetworks = []api.NetworksPost{lxd.DefaultPendingFanNetwork()} - if peer == sh.Name { - network, err := lxd.DefaultFanNetwork() - if err != nil { - return err - } - system.Networks = []api.NetworksPost{network} + // Check if FAN networking is usable. + fanUsable, _, err := lxd.FanNetworkUsable() + if err != nil { + return err + } + + if fanUsable { + for peer, system := range systems { + if bootstrap { + system.TargetNetworks = []api.NetworksPost{lxd.DefaultPendingFanNetwork()} + if peer == sh.Name { + network, err := lxd.DefaultFanNetwork() + if err != nil { + return err + } + + system.Networks = []api.NetworksPost{network} + } } - } - systems[peer] = system + systems[peer] = system + } } // Automatic setup gets a basic fan setup. if autoSetup { + if !fanUsable { + fmt.Println("FAN networking is not usable, skipping") + } + return nil } // Environments without OVN get a basic fan setup. if sh.Services[types.MicroOVN] == nil { + if !fanUsable { + fmt.Println("FAN networking is not usable, skipping") + } + return nil } @@ -769,6 +786,10 @@ func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSyst if !canOVN { fmt.Println("No dedicated uplink interfaces detected, skipping distributed networking") + if !fanUsable { + fmt.Println("FAN networking is not usable, skipping") + } + return nil } diff --git a/cmd/microcloud/main_init_preseed.go b/cmd/microcloud/main_init_preseed.go index 15ddac7e4..9e6c396af 100644 --- a/cmd/microcloud/main_init_preseed.go +++ b/cmd/microcloud/main_init_preseed.go @@ -498,8 +498,14 @@ func (p *Preseed) Parse(s *service.Handler, bootstrap bool) (map[string]InitSyst systems[peer] = system } + // Check if FAN networking is usable. + fanUsable, _, err := lxd.FanNetworkUsable() + if err != nil { + return nil, err + } + // Setup FAN network if OVN not available. - if len(ifaceByPeer) == 0 { + if len(ifaceByPeer) == 0 && fanUsable { for peer, system := range systems { if bootstrap { system.TargetNetworks = append(system.TargetNetworks, lxd.DefaultPendingFanNetwork()) From 459de0bb85933c4fe046319e97bcb925818a5c2a Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Mon, 10 Jun 2024 10:00:11 +0200 Subject: [PATCH 6/7] cmd/microcloud: Add `askProceedIfNoOverlayNetwork` question wheen no overlay networking solution is available Signed-off-by: Gabriel Mougard --- cmd/microcloud/ask.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index 4ef23f05d..b56ee7db2 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -625,6 +625,19 @@ func (c *CmdControl) askRemotePool(systems map[string]InitSystem, autoSetup bool return nil } +func (c *CmdControl) askProceedIfNoOverlayNetwork() error { + proceedWithNoOverlayNetworking, err := c.asker.AskBool("FAN networking is not usable. Do you want to proceed with setting up an inoperable cluster? (yes/no) [default=no]: ", "no") + if err != nil { + return err + } + + if proceedWithNoOverlayNetworking { + return nil + } + + return fmt.Errorf("cluster bootstrapping aborted due to lack of usable networking") +} + func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSystem, microCloudInternalSubnet *net.IPNet, autoSetup bool) error { _, bootstrap := systems[sh.Name] lxd := sh.Services[types.LXD].(*service.LXDService) @@ -656,7 +669,7 @@ func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSyst // Automatic setup gets a basic fan setup. if autoSetup { if !fanUsable { - fmt.Println("FAN networking is not usable, skipping") + return c.askProceedIfNoOverlayNetwork() } return nil @@ -665,7 +678,7 @@ func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSyst // Environments without OVN get a basic fan setup. if sh.Services[types.MicroOVN] == nil { if !fanUsable { - fmt.Println("FAN networking is not usable, skipping") + return c.askProceedIfNoOverlayNetwork() } return nil @@ -787,7 +800,7 @@ func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSyst if !canOVN { fmt.Println("No dedicated uplink interfaces detected, skipping distributed networking") if !fanUsable { - fmt.Println("FAN networking is not usable, skipping") + return c.askProceedIfNoOverlayNetwork() } return nil From 8f91864dbdccd100cbd49d491c00f48671f1628b Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Fri, 24 May 2024 10:12:44 +0200 Subject: [PATCH 7/7] tests: Add an interactive MicroCloud test with IPv6 disabled and IPv4 disabled Signed-off-by: Gabriel Mougard --- test/includes/microcloud.sh | 42 ++++++++++++++++++++++--------------- test/suites/basic.sh | 40 +++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh index 93f53ef7b..332541578 100644 --- a/test/includes/microcloud.sh +++ b/test/includes/microcloud.sh @@ -5,7 +5,7 @@ unset_interactive_vars() { unset LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \ SETUP_ZFS ZFS_FILTER ZFS_WIPE \ SETUP_CEPH CEPH_WARNING CEPH_FILTER CEPH_WIPE SETUP_CEPHFS CEPH_CLUSTER_NETWORK IGNORE_CEPH_NETWORKING \ - SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET + PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET } # microcloud_interactive: outputs text that can be passed to `TEST_CONSOLE=1 microcloud init` @@ -19,24 +19,25 @@ microcloud_interactive() { EXPECT_PEERS=${EXPECT_PEERS:-} # wait for this number of systems to be available to join the cluster. REUSE_EXISTING=${REUSE_EXISTING:-} # (yes/no) incorporate an existing clustered service. REUSE_EXISTING_COUNT=${REUSE_EXISTING_COUNT:-0} # (number) number of existing clusters to incorporate. - SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup. - ZFS_FILTER=${ZFS_FILTER:-} # filter string for ZFS disks. - ZFS_WIPE=${ZFS_WIPE:-} # (yes/no) to wipe all disks. - SETUP_CEPH=${SETUP_CEPH:-} # (yes/no) input for initiating CEPH storage pool setup. - SETUP_CEPHFS=${SETUP_CEPHFS:-} # (yes/no) input for initialising CephFS storage pool setup. - CEPH_WARNING=${CEPH_WARNING:-} # (yes/no) input for warning about eligible disk detection. - CEPH_FILTER=${CEPH_FILTER:-} # filter string for CEPH disks. - CEPH_WIPE=${CEPH_WIPE:-} # (yes/no) to wipe all disks. + SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup. + ZFS_FILTER=${ZFS_FILTER:-} # filter string for ZFS disks. + ZFS_WIPE=${ZFS_WIPE:-} # (yes/no) to wipe all disks. + SETUP_CEPH=${SETUP_CEPH:-} # (yes/no) input for initiating CEPH storage pool setup. + SETUP_CEPHFS=${SETUP_CEPHFS:-} # (yes/no) input for initialising CephFS storage pool setup. + CEPH_WARNING=${CEPH_WARNING:-} # (yes/no) input for warning about eligible disk detection. + CEPH_FILTER=${CEPH_FILTER:-} # filter string for CEPH disks. + CEPH_WIPE=${CEPH_WIPE:-} # (yes/no) to wipe all disks. CEPH_CLUSTER_NETWORK=${CEPH_CLUSTER_NETWORK:-} # (default: MicroCloud internal subnet or Ceph public network if specified previously) input for setting up a cluster network. IGNORE_CEPH_NETWORKING=${IGNORE_CEPH_NETWORKING:-} # (yes/no) input for ignoring Ceph network setup. Set it to `yes` during `microcloud add` . - SETUP_OVN=${SETUP_OVN:-} # (yes/no) input for initiating OVN network setup. - OVN_WARNING=${OVN_WARNING:-} # (yes/no) input for warning about eligible interface detection. - OVN_FILTER=${OVN_FILTER:-} # filter string for OVN interfaces. - IPV4_SUBNET=${IPV4_SUBNET:-} # OVN ipv4 gateway subnet. - IPV4_START=${IPV4_START:-} # OVN ipv4 range start. - IPV4_END=${IPV4_END:-} # OVN ipv4 range end. - DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses. - IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range. + PROCEED_WITH_NO_OVERLAY_NETWORKING=${PROCEED_WITH_NO_OVERLAY_NETWORKING:-} # (yes/no) input for proceeding without overlay networking. + SETUP_OVN=${SETUP_OVN:-} # (yes/no) input for initiating OVN network setup. + OVN_WARNING=${OVN_WARNING:-} # (yes/no) input for warning about eligible interface detection. + OVN_FILTER=${OVN_FILTER:-} # filter string for OVN interfaces. + IPV4_SUBNET=${IPV4_SUBNET:-} # OVN ipv4 gateway subnet. + IPV4_START=${IPV4_START:-} # OVN ipv4 range start. + IPV4_END=${IPV4_END:-} # OVN ipv4 range end. + DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses. + IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range. setup=" ${LOOKUP_IFACE} # filter the lookup interface @@ -88,6 +89,13 @@ $(true) # workaround for set -e " fi +if [ -n "${PROCEED_WITH_NO_OVERLAY_NETWORKING}" ]; then + setup="${setup} +${PROCEED_WITH_NO_OVERLAY_NETWORKING} # agree to proceed without overlay networking (neither FAN nor OVN networking) (yes/no) +$(true) # workaround for set -e +" +fi + if [ -z "${IGNORE_CEPH_NETWORKING}" ]; then if [ -n "${CEPH_CLUSTER_NETWORK}" ]; then setup="${setup} diff --git a/test/suites/basic.sh b/test/suites/basic.sh index 5d636f0c1..edd42aad3 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -47,6 +47,46 @@ test_interactive() { validate_system_lxd "${m}" 3 disk1 done + # Reset the systems with just LXD and no IPv6 support. + reset_systems 3 3 1 + + for m in micro01 micro02 micro03 ; do + lxc exec "${m}" -- echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6 + lxc exec "${m}" -- snap disable microceph || true + lxc exec "${m}" -- snap disable microovn || true + lxc exec "${m}" -- snap restart microcloud + done + + echo "Creating a MicroCloud with ZFS storage and no IPv6 support" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + for m in micro01 micro02 micro03 ; do + validate_system_lxd "${m}" 3 disk1 + done + + # Reset the systems with just LXD and no IPv4 support. + gw_net_addr=$(lxc network get lxdbr0 ipv4.address) + lxc network set lxdbr0 ipv4.address none + reset_systems 3 3 1 + + for m in micro01 micro02 micro03 ; do + lxc exec "${m}" -- snap disable microceph || true + lxc exec "${m}" -- snap disable microovn || true + lxc exec "${m}" -- snap restart microcloud + done + + export PROCEED_WITH_NO_OVERLAY_NETWORKING="no" # This will avoid to setup the cluster if no overlay networking is available. + echo "Creating a MicroCloud with ZFS storage and no IPv4 support" + ! microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init 2> err" || false + + # Ensure we error out due to a lack of usable overlay networking. + lxc exec micro01 -- cat err | grep "cluster bootstrapping aborted due to lack of usable networking" -q + + # Set the IPv4 address back to the original value. + lxc network set lxdbr0 ipv4.address "${gw_net_addr}" + unset PROCEED_WITH_NO_OVERLAY_NETWORKING + # Reset the systems and install microceph. reset_systems 3 3 1