diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index a7c2131da..662082e1c 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -176,26 +176,38 @@ func (c *CmdControl) askDisks(sh *service.Handler, systems map[string]InitSystem systems[peer] = system } - wantsDisks := true - if !autoSetup && foundDisks { - wantsDisks, err = c.asker.AskBool("Would you like to set up local storage? (yes/no) [default=yes]: ", "yes") - if err != nil { - return err - } + lxd := sh.Services[types.LXD].(*service.LXDService) + client, err := lxd.Client(context.Background(), "") + if err != nil { + return err } - if !foundDisks { - wantsDisks = false + storagePools, err := client.GetStoragePoolNames() + if err != nil { + return err } - lxd := sh.Services[types.LXD].(*service.LXDService) - if wantsDisks { - c.askRetry("Retry selecting disks?", autoSetup, func() error { - return askLocalPool(systems, autoSetup, wipeAllDisks, *lxd) - }) + wantsDisks := true + if !shared.ValueInSlice[string](lxd.DefaultZFSStoragePool().Name, storagePools) { + if !autoSetup && foundDisks { + wantsDisks, err = c.asker.AskBool("Would you like to set up local storage? (yes/no) [default=yes]: ", "yes") + if err != nil { + return err + } + } + + if !foundDisks { + wantsDisks = false + } + + if wantsDisks { + c.askRetry("Retry selecting disks?", autoSetup, func() error { + return askLocalPool(systems, autoSetup, wipeAllDisks, *lxd) + }) + } } - if sh.Services[types.MicroCeph] != nil { + if sh.Services[types.MicroCeph] != nil && !shared.ValueInSlice[string](lxd.DefaultCephStoragePool().Name, storagePools) { availableDisks := map[string][]api.ResourcesStorageDisk{} for peer, system := range systems { if len(system.AvailableDisks) > 0 { @@ -628,20 +640,38 @@ func (c *CmdControl) askRemotePool(systems map[string]InitSystem, autoSetup bool func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSystem, microCloudInternalSubnet *net.IPNet, autoSetup bool) error { _, bootstrap := systems[sh.Name] lxd := sh.Services[types.LXD].(*service.LXDService) - for peer, system := range systems { - if bootstrap { - system.TargetNetworks = []api.NetworksPost{lxd.DefaultPendingFanNetwork()} - if peer == sh.Name { - network, err := lxd.DefaultFanNetwork() - if err != nil { - return err - } - system.Networks = []api.NetworksPost{network} + lxdClient, err := lxd.Client(context.Background(), "") + if err != nil { + return err + } + + networkNames, err := lxdClient.GetNetworkNames() + if err != nil { + return err + } + + networkExists := make(map[string]bool, len(networkNames)) + for _, net := range networkNames { + networkExists[net] = true + } + + if !networkExists[lxd.DefaultPendingFanNetwork().Name] { + for peer, system := range systems { + if bootstrap { + system.TargetNetworks = []api.NetworksPost{lxd.DefaultPendingFanNetwork()} + if peer == sh.Name { + network, err := lxd.DefaultFanNetwork() + if err != nil { + return err + } + + system.Networks = []api.NetworksPost{network} + } } - } - systems[peer] = system + systems[peer] = system + } } // Automatic setup gets a basic fan setup. @@ -650,7 +680,8 @@ func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSyst } // Environments without OVN get a basic fan setup. - if sh.Services[types.MicroOVN] == nil { + uplink, ovn := lxd.DefaultOVNNetwork("", "", "", "") + if sh.Services[types.MicroOVN] == nil || networkExists[uplink.Name] || networkExists[ovn.Name] { return nil } diff --git a/cmd/microcloud/main.go b/cmd/microcloud/main.go index 19519492d..0af5673ba 100644 --- a/cmd/microcloud/main.go +++ b/cmd/microcloud/main.go @@ -81,6 +81,9 @@ EOF`) var cmdAdd = cmdAdd{common: &commonCmd} app.AddCommand(cmdAdd.Command()) + var cmdService = cmdServices{common: &commonCmd} + app.AddCommand(cmdService.Command()) + var cmdPeers = cmdClusterMembers{common: &commonCmd} app.AddCommand(cmdPeers.Command()) diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index 074207038..01b7467af 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -933,12 +933,34 @@ func setupCluster(s *service.Handler, systems map[string]InitSystem) error { if !shared.ValueInSlice(profile.Name, profiles) { err = lxdClient.CreateProfile(profile) + if err != nil { + return err + } } else { - err = lxdClient.UpdateProfile(profile.Name, profile.ProfilePut, "") - } + // Ensure any pre-existing devices and config are carried over to the new profile, unless we are managing them. + existingProfile, _, err := lxdClient.GetProfile("default") + if err != nil { + return err + } - if err != nil { - return err + for k, v := range existingProfile.Config { + _, ok := profile.Config[k] + if !ok { + profile.Config[k] = v + } + } + + for k, v := range existingProfile.Devices { + _, ok := profile.Devices[k] + if !ok { + profile.Devices[k] = v + } + } + + err = lxdClient.UpdateProfile(profile.Name, profile.ProfilePut, "") + if err != nil { + return err + } } } diff --git a/cmd/microcloud/services.go b/cmd/microcloud/services.go new file mode 100644 index 000000000..c471bc132 --- /dev/null +++ b/cmd/microcloud/services.go @@ -0,0 +1,341 @@ +package main + +import ( + "context" + "fmt" + "net" + "sort" + "strings" + "sync" + + "github.com/canonical/lxd/client" + "github.com/canonical/lxd/shared" + cli "github.com/canonical/lxd/shared/cmd" + "github.com/canonical/microcluster/client" + "github.com/canonical/microcluster/microcluster" + "github.com/spf13/cobra" + + "github.com/canonical/microcloud/microcloud/api" + "github.com/canonical/microcloud/microcloud/api/types" + "github.com/canonical/microcloud/microcloud/mdns" + "github.com/canonical/microcloud/microcloud/service" +) + +type cmdServices struct { + common *CmdControl +} + +func (c *cmdServices) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "service", + Short: "Manage MicroCloud services", + RunE: func(cmd *cobra.Command, args []string) error { return cmd.Help() }, + } + + var cmdServiceList = cmdServiceList{common: c.common} + cmd.AddCommand(cmdServiceList.Command()) + + var cmdServiceAdd = cmdServiceAdd{common: c.common} + cmd.AddCommand(cmdServiceAdd.Command()) + + return cmd +} + +type cmdServiceList struct { + common *CmdControl +} + +func (c *cmdServiceList) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List MicroCloud services and their cluster members", + RunE: c.Run, + } + + return cmd +} + +func (c *cmdServiceList) Run(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return cmd.Help() + } + + // Get a microcluster client so we can get state information. + cloudApp, err := microcluster.App(microcluster.Args{StateDir: c.common.FlagMicroCloudDir}) + if err != nil { + return err + } + + // Fetch the name and address, and ensure we're initialized. + status, err := cloudApp.Status(context.Background()) + if err != nil { + return fmt.Errorf("Failed to get MicroCloud status: %w", err) + } + + if !status.Ready { + return fmt.Errorf("MicroCloud is uninitialized, run 'microcloud init' first") + } + + services := []types.ServiceType{types.MicroCloud, types.LXD} + optionalServices := map[types.ServiceType]string{ + types.MicroCeph: api.MicroCephDir, + types.MicroOVN: api.MicroOVNDir, + } + + services, err = c.common.askMissingServices(services, optionalServices, true) + if err != nil { + return err + } + + // Instantiate a handler for the services. + s, err := service.NewHandler(status.Name, status.Address.Addr().String(), c.common.FlagMicroCloudDir, c.common.FlagLogDebug, c.common.FlagLogVerbose, services...) + if err != nil { + return err + } + + mu := sync.Mutex{} + header := []string{"NAME", "ADDRESS", "ROLE", "STATUS"} + allClusters := map[types.ServiceType][][]string{} + err = s.RunConcurrent(false, false, func(s service.Service) error { + var err error + var data [][]string + var microClient *client.Client + var lxd lxd.InstanceServer + switch s.Type() { + case types.LXD: + lxd, err = s.(*service.LXDService).Client(context.Background(), "") + case types.MicroCeph: + microClient, err = s.(*service.CephService).Client("", "") + case types.MicroOVN: + microClient, err = s.(*service.OVNService).Client() + case types.MicroCloud: + microClient, err = s.(*service.CloudService).Client() + } + + if err != nil { + return err + } + + if microClient != nil { + clusterMembers, err := microClient.GetClusterMembers(context.Background()) + if err != nil && err.Error() != "Daemon not yet initialized" { + return err + } + + if len(clusterMembers) != 0 { + data = make([][]string, len(clusterMembers)) + for i, clusterMember := range clusterMembers { + data[i] = []string{clusterMember.Name, clusterMember.Address.String(), clusterMember.Role, string(clusterMember.Status)} + } + + sort.Sort(cli.SortColumnsNaturally(data)) + } + } else if lxd != nil { + clusterMembers, err := lxd.GetClusterMembers() + if err != nil { + return err + } + + data = make([][]string, len(clusterMembers)) + for i, clusterMember := range clusterMembers { + data[i] = []string{clusterMember.ServerName, clusterMember.URL, strings.Join(clusterMember.Roles, "\n"), string(clusterMember.Status)} + } + + sort.Sort(cli.SortColumnsNaturally(data)) + } + + mu.Lock() + allClusters[s.Type()] = data + mu.Unlock() + + return nil + }) + if err != nil { + return err + } + + for serviceType, data := range allClusters { + if len(data) == 0 { + fmt.Printf("%s: Not initialized\n", serviceType) + } else { + fmt.Printf("%s:\n", serviceType) + err = cli.RenderTable(cli.TableFormatTable, header, data, nil) + if err != nil { + return err + } + } + } + + return nil +} + +type cmdServiceAdd struct { + common *CmdControl +} + +func (c *cmdServiceAdd) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "add", + Short: "Set up new services on the existing MicroCloud", + RunE: c.Run, + } + + return cmd +} + +func (c *cmdServiceAdd) Run(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return cmd.Help() + } + + // Get a microcluster client so we can get state information. + cloudApp, err := microcluster.App(microcluster.Args{StateDir: c.common.FlagMicroCloudDir}) + if err != nil { + return err + } + + // Fetch the name and address, and ensure we're initialized. + status, err := cloudApp.Status(context.Background()) + if err != nil { + return fmt.Errorf("Failed to get MicroCloud status: %w", err) + } + + if !status.Ready { + return fmt.Errorf("MicroCloud is uninitialized, run 'microcloud init' first") + } + + services := []types.ServiceType{types.MicroCloud, types.LXD} + optionalServices := map[types.ServiceType]string{ + types.MicroCeph: api.MicroCephDir, + types.MicroOVN: api.MicroOVNDir, + } + + // Set the auto flag to true so that we automatically omit any services that aren't installed. + services, err = c.common.askMissingServices(services, optionalServices, true) + if err != nil { + return err + } + + // Instantiate a handler for the services. + s, err := service.NewHandler(status.Name, status.Address.Addr().String(), c.common.FlagMicroCloudDir, c.common.FlagLogDebug, c.common.FlagLogVerbose, services...) + if err != nil { + return err + } + + // Fetch the cluster members for services we want to ignore. + cloudCluster, err := s.Services[types.MicroCloud].ClusterMembers(context.Background()) + if err != nil { + return fmt.Errorf("Failed to inspect existing cluster: %w", err) + } + + lxdCluster, err := s.Services[types.LXD].ClusterMembers(context.Background()) + if err != nil { + return fmt.Errorf("Failed to inspect existing cluster: %w", err) + } + + // Create an InitSystem map to carry through the interactive setup. + systems := make(map[string]InitSystem, len(cloudCluster)) + for name, address := range cloudCluster { + host, _, err := net.SplitHostPort(address) + if err != nil { + return fmt.Errorf("Failed to parse cluster member address %q: %w", address, err) + } + + systems[name] = InitSystem{ + ServerInfo: mdns.ServerInfo{ + Name: name, + Address: host, + Services: services, + }, + InitializedServices: map[types.ServiceType]map[string]string{ + types.LXD: lxdCluster, + types.MicroCloud: cloudCluster, + }, + } + } + + // Check if there are any pre-existing clusters that we can re-use for each optional service. + availableServices := map[types.ServiceType]string{} + for _, service := range services { + if service == types.LXD || service == types.MicroCloud { + continue + } + + // Get the first system that has initialized an optional service, and its list of cluster members. We may or may not already be in this cluster. + firstSystem, clusterMembers, err := checkClustered(s, false, service, systems) + if err != nil { + return err + } + + // If no system is clustered yet, record that too so we can try to set it up. + if firstSystem == "" { + availableServices[service] = "" + continue + } + + // If any service has all of the cluster members recorded on the MicroCloud daemon already, + // then it can be considered part of the microcloud already, so we can ignore it. + allMembersExist := true + for name := range cloudCluster { + _, ok := clusterMembers[name] + if !ok { + allMembersExist = false + break + } + } + + if !allMembersExist { + availableServices[service] = firstSystem + } + } + + // Ask to reuse or skip existing clusters. + for serviceType, system := range availableServices { + question := fmt.Sprintf("%q is already part of a %s cluster. Use this cluster with MicroCloud, or skip %s? (reuse/skip) [default=reuse]", system, serviceType, serviceType) + validator := func(s string) error { + if !shared.ValueInSlice(s, []string{"reuse", "skip"}) { + return fmt.Errorf("Invalid input, expected one of (reuse,skip) but got %q", s) + } + + return nil + } + + if system == "" { + continue + } + + reuseOrSkip, err := c.common.asker.AskString(question, "reuse", validator) + if err != nil { + return err + } + + if reuseOrSkip != "reuse" { + delete(s.Services, serviceType) + delete(availableServices, serviceType) + } + } + + // Go through the normal setup for disks and networks if necessary. + _, ok := availableServices[types.MicroCeph] + if ok { + err = c.common.askDisks(s, systems, false, false) + if err != nil { + return err + } + } + + _, _, subnet, err := c.common.askAddress(true, status.Address.Addr().String()) + if err != nil { + return err + } + + _, ok = availableServices[types.MicroOVN] + if ok { + err = c.common.askNetwork(s, systems, subnet, false) + if err != nil { + return err + } + } + + return setupCluster(s, systems) +} diff --git a/doc/how-to/add_service.rst b/doc/how-to/add_service.rst new file mode 100644 index 000000000..a3a15e813 --- /dev/null +++ b/doc/how-to/add_service.rst @@ -0,0 +1,41 @@ +.. _howto-add-service: + +How to add a new service +======================== + +If you set up the MicroCloud without MicroOVN or MicroCeph initially, you can add those services with the command :command:`microcloud service add`:: + + sudo microcloud service add + +If MicroCloud detects a service is installed but not set up, it will ask to configure the service. + +#. Select whether you want to set up distributed storage (if adding MicroCeph to the MicroCloud). + + .. note:: + To set up distributed storage, you need at least three additional disks on at least three different machines. + The disks must not contain any partitions. + + If you choose ``yes``, configure the distributed storage: + + 1. Select the disks that you want to use for distributed storage. + + You must select at least three disks. + #. Select whether you want to wipe any of the disks. + Wiping a disk will destroy all data on it. + + #. You can choose to optionally set up a CephFS distributed file system. + +#. Select either an IPv4 or IPv6 CIDR subnet for the Ceph internal traffic. You can leave it empty to use the default value, which is the MicroCloud internal network (see :ref:`howto-ceph-networking` for how to configure it). + +#. Select whether you want to set up distributed networking (if adding MicroOVN to the MicroCloud). + + If you choose ``yes``, configure the distributed networking: + + 1. Select the network interfaces that you want to use (see :ref:`microcloud-networking-uplink`). + + You must select one network interface per machine. + #. If you want to use IPv4, specify the IPv4 gateway on the uplink network (in CIDR notation) and the first and last IPv4 address in the range that you want to use with LXD. + #. If you want to use IPv6, specify the IPv6 gateway on the uplink network (in CIDR notation). +#. MicroCloud now starts to bootstrap the cluster for only the new services. + Monitor the output to see whether all steps complete successfully. + See :ref:`bootstrapping-process` for more information. diff --git a/doc/how-to/commands.rst b/doc/how-to/commands.rst index 39fe53e2a..37596c3a6 100644 --- a/doc/how-to/commands.rst +++ b/doc/how-to/commands.rst @@ -259,7 +259,10 @@ See :ref:`lxd:cluster-manage-instance` and :ref:`lxd:cluster-evacuate`. .. list-table:: :widths: 2 3 - * - Inspect the cluster status + * - Inspect the cluster status for all services at once + - :command:`microcloud service list` + + * - Inspect the cluster status for each service - :command:`microcloud cluster list` :command:`lxc cluster list` diff --git a/doc/how-to/index.rst b/doc/how-to/index.rst index 81ff2ae38..92908e86e 100644 --- a/doc/how-to/index.rst +++ b/doc/how-to/index.rst @@ -13,6 +13,7 @@ These how-to guides cover key operations and processes in MicroCloud. Initialise MicroCloud Configure Ceph networking Add a machine + Add a service Get support Contribute to MicroCloud Work with MicroCloud diff --git a/service/lxd.go b/service/lxd.go index 009c256bc..1b366fbe1 100644 --- a/service/lxd.go +++ b/service/lxd.go @@ -71,9 +71,16 @@ func (s LXDService) remoteClient(secret string, address string, port int64) (lxd return nil, err } + serverCert, err := s.m.FileSystem.ServerCert() + if err != nil { + return nil, err + } + remoteURL := c.URL() client, err := lxd.ConnectLXD(remoteURL.String(), &lxd.ConnectionArgs{ HTTPClient: c.Client.Client, + TLSClientCert: string(serverCert.PublicKey()), + TLSClientKey: string(serverCert.PrivateKey()), InsecureSkipVerify: true, SkipGetServer: true, Proxy: cloudClient.AuthProxy(secret, types.LXD), diff --git a/service/microcloud.go b/service/microcloud.go index 40af0ef10..d99710c15 100644 --- a/service/microcloud.go +++ b/service/microcloud.go @@ -53,6 +53,11 @@ func NewCloudService(name string, addr string, dir string, verbose bool, debug b }, nil } +// Client returns a client to the MicroCloud unix socket. +func (s CloudService) Client() (*microClient.Client, error) { + return s.client.LocalClient() +} + // StartCloud launches the MicroCloud daemon with the appropriate hooks. func (s *CloudService) StartCloud(ctx context.Context, service *Handler, endpoints []rest.Endpoint) error { return s.client.Start(ctx, endpoints, nil, nil, &config.Hooks{ diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh index 7011bdf47..aa3461e60 100644 --- a/test/includes/microcloud.sh +++ b/test/includes/microcloud.sh @@ -5,7 +5,8 @@ unset_interactive_vars() { unset LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \ SETUP_ZFS ZFS_FILTER ZFS_WIPE \ SETUP_CEPH CEPH_WARNING CEPH_FILTER CEPH_WIPE SETUP_CEPHFS CEPH_CLUSTER_NETWORK IGNORE_CEPH_NETWORKING \ - SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET + SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \ + SKIP_LOOKUP } # microcloud_interactive: outputs text that can be passed to `TEST_CONSOLE=1 microcloud init` @@ -13,6 +14,7 @@ unset_interactive_vars() { # The lines that are output are based on the values passed to the listed environment variables. # Any unset variables will be omitted. microcloud_interactive() { + SKIP_LOOKUP=${SKIP_LOOKUP:-} # whether or not to skip the whole lookup block in the interactive command list. LOOKUP_IFACE=${LOOKUP_IFACE:-} # filter string for the lookup interface table. LIMIT_SUBNET=${LIMIT_SUBNET:-} # (yes/no) input for limiting lookup of systems to the above subnet. SKIP_SERVICE=${SKIP_SERVICE:-} # (yes/no) input to skip any missing services. Should be unset if all services are installed. @@ -38,7 +40,9 @@ microcloud_interactive() { DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses. IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range. - setup=" + setup="" + if ! [ "${SKIP_LOOKUP}" = 1 ]; then + setup=" ${LOOKUP_IFACE} # filter the lookup interface $([ -n "${LOOKUP_IFACE}" ] && printf "select") # select the interface $([ -n "${LOOKUP_IFACE}" ] && printf -- "---") @@ -49,14 +53,14 @@ select-all # select all the sys --- $(true) # workaround for set -e " + fi if [ -n "${REUSE_EXISTING}" ]; then for i in $(seq 1 "${REUSE_EXISTING_COUNT}") ; do - setup=$(cat << EOF -${setup} + setup="${setup} ${REUSE_EXISTING} -EOF -) +$(true) # workaround for set -e +" done fi diff --git a/test/main.sh b/test/main.sh index cb036b218..515baa806 100755 --- a/test/main.sh +++ b/test/main.sh @@ -185,6 +185,7 @@ run_instances_tests() { run_basic_tests() { run_test test_reuse_cluster "reuse_cluster" run_test test_auto "auto" + run_test test_add_services "add services" } run_interactive_tests() { diff --git a/test/suites/basic.sh b/test/suites/basic.sh index 5d636f0c1..fd09f719e 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -1147,3 +1147,100 @@ EOF services_validator } + +test_add_services() { + unset_interactive_vars + # Set the default config for interactive setup. + export LOOKUP_IFACE="enp5s0" + export LIMIT_SUBNET="yes" + export EXPECT_PEERS=2 + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="yes" + export SETUP_CEPHFS="yes" + export CEPH_FILTER="lxd_disk2" + export CEPH_WIPE="yes" + export SETUP_OVN="yes" + export OVN_FILTER="enp6s0" + export IPV4_SUBNET="10.1.123.1/24" + export IPV4_START="10.1.123.100" + export IPV4_END="10.1.123.254" + export CUSTOM_DNS_ADDRESSES="10.1.123.1,8.8.8.8" + export IPV6_SUBNET="fd42:1:1234:1234::1/64" + + reset_systems 3 3 3 + echo Add MicroCeph to MicroCloud that was set up without it, and setup remote storage + lxc exec micro01 -- snap disable microceph + unset SETUP_CEPH + export SKIP_SERVICE="yes" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + lxc exec micro01 -- snap enable microceph + export SETUP_CEPH="yes" + export SKIP_LOOKUP=1 + unset SETUP_ZFS + unset SETUP_OVN + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" + services_validator + + reset_systems 3 3 3 + echo Add MicroOVN to MicroCloud that was set up without it, and setup ovn network + lxc exec micro01 -- snap disable microovn + export SETUP_ZFS="yes" + unset SKIP_LOOKUP + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + lxc exec micro01 -- snap enable microovn + export SETUP_OVN="yes" + export SKIP_LOOKUP=1 + unset SETUP_ZFS + unset SETUP_CEPH + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" + services_validator + + reset_systems 3 3 3 + echo Add both MicroOVN and MicroCeph to a MicroCloud that was set up without it + lxc exec micro01 -- snap disable microovn + lxc exec micro01 -- snap disable microceph + export SETUP_ZFS="yes" + unset SKIP_LOOKUP + unset SETUP_OVN + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + lxc exec micro01 -- snap enable microovn + lxc exec micro01 -- snap enable microceph + export SETUP_OVN="yes" + export SETUP_CEPH="yes" + export SKIP_LOOKUP=1 + unset SETUP_ZFS + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" + services_validator + + reset_systems 3 3 3 + echo Reuse a MicroCeph that was set up on one node of the MicroCloud + lxc exec micro01 -- snap disable microceph + lxc exec micro02 -- microceph cluster bootstrap + export SETUP_ZFS="yes" + unset SETUP_CEPH + unset SKIP_LOOKUP + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + lxc exec micro01 -- snap enable microceph + export REUSE_EXISTING_COUNT=1 + export REUSE_EXISTING="reuse" + export SETUP_CEPH="yes" + export SKIP_LOOKUP=1 + unset SETUP_ZFS + unset SETUP_OVN + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" + services_validator + + reset_systems 3 3 3 + echo Fail to add any services if they have been set up + export SETUP_ZFS="yes" + export SETUP_OVN="yes" + unset REUSE_EXISTING + unset REUSE_EXISTING_COUNT + unset SKIP_LOOKUP + unset SKIP_SERVICE + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + export SKIP_LOOKUP=1 + ! microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" || true +}