diff --git a/cmd/microcloud/add.go b/cmd/microcloud/add.go index 780a2e93..7c52d389 100644 --- a/cmd/microcloud/add.go +++ b/cmd/microcloud/add.go @@ -141,7 +141,7 @@ func (c *cmdAdd) Run(cmd *cobra.Command, args []string) error { } // Ask to reuse existing clusters. - err = cfg.askClustered(s) + err = cfg.askClustered(s, services) if err != nil { return err } diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index d11b3ece..6fdf140a 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -1230,13 +1230,8 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error { // If a service is already initialized on some systems, we will offer to add the remaining systems, or skip that service. // In auto setup, we will expect no initialized services so that we can be opinionated about how we configure the cluster without user input. // This works by deleting the record for the service from the `service.Handler`, thus ignoring it for the remainder of the setup. -func (c *initConfig) askClustered(s *service.Handler) error { - expectedServices := make(map[types.ServiceType]struct{}, len(s.Services)) - for k := range s.Services { - expectedServices[k] = struct{}{} - } - - for serviceType := range expectedServices { +func (c *initConfig) askClustered(s *service.Handler, expectedServices []types.ServiceType) error { + for _, serviceType := range expectedServices { for name, info := range c.state { _, newSystem := c.systems[name] if !newSystem { diff --git a/cmd/microcloud/main.go b/cmd/microcloud/main.go index 76d4b2df..4244e77a 100644 --- a/cmd/microcloud/main.go +++ b/cmd/microcloud/main.go @@ -84,6 +84,9 @@ EOF`) var cmdRemove = cmdRemove{common: &commonCmd} app.AddCommand(cmdRemove.Command()) + var cmdService = cmdServices{common: &commonCmd} + app.AddCommand(cmdService.Command()) + var cmdPeers = cmdClusterMembers{common: &commonCmd} app.AddCommand(cmdPeers.Command()) diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index fd8f49f0..a9d67496 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -244,7 +244,7 @@ func (c *initConfig) RunInteractive(cmd *cobra.Command, args []string) error { } // Ask to reuse existing clusters. - err = c.askClustered(s) + err = c.askClustered(s, services) if err != nil { return err } @@ -952,12 +952,57 @@ func (c *initConfig) setupCluster(s *service.Handler) error { if !shared.ValueInSlice(profile.Name, profiles) { err = lxdClient.CreateProfile(profile) + if err != nil { + return err + } } else { - err = lxdClient.UpdateProfile(profile.Name, profile.ProfilePut, "") - } + // Ensure any pre-existing devices and config are carried over to the new profile, unless we are managing them. + existingProfile, _, err := lxdClient.GetProfile("default") + if err != nil { + return err + } - if err != nil { - return err + askConflictingConfig := []string{} + askConflictingDevices := []string{} + for k, v := range profile.Config { + _, ok := existingProfile.Config[k] + if !ok { + existingProfile.Config[k] = v + } else { + askConflictingConfig = append(askConflictingConfig, k) + } + } + + for k, v := range profile.Devices { + _, ok := existingProfile.Devices[k] + if !ok { + existingProfile.Devices[k] = v + } else { + askConflictingDevices = append(askConflictingDevices, k) + } + } + + if !c.autoSetup && len(askConflictingConfig) > 0 || len(askConflictingDevices) > 0 { + replace, err := c.asker.AskBool("Replace existing default profile configuration? (yes/no) [default=no]: ", "no") + if err != nil { + return err + } + + if replace { + for _, key := range askConflictingConfig { + existingProfile.Config[key] = profile.Config[key] + } + + for _, key := range askConflictingDevices { + existingProfile.Devices[key] = profile.Devices[key] + } + } + } + + err = lxdClient.UpdateProfile(profile.Name, existingProfile.Writable(), "") + if err != nil { + return err + } } } diff --git a/cmd/microcloud/services.go b/cmd/microcloud/services.go new file mode 100644 index 00000000..4387927a --- /dev/null +++ b/cmd/microcloud/services.go @@ -0,0 +1,339 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "sort" + "strings" + "sync" + + "github.com/canonical/lxd/client" + lxdAPI "github.com/canonical/lxd/shared/api" + cli "github.com/canonical/lxd/shared/cmd" + "github.com/canonical/microcluster/client" + "github.com/canonical/microcluster/microcluster" + "github.com/spf13/cobra" + + "github.com/canonical/microcloud/microcloud/api" + "github.com/canonical/microcloud/microcloud/api/types" + "github.com/canonical/microcloud/microcloud/mdns" + "github.com/canonical/microcloud/microcloud/service" +) + +type cmdServices struct { + common *CmdControl +} + +func (c *cmdServices) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "service", + Short: "Manage MicroCloud services", + RunE: func(cmd *cobra.Command, args []string) error { return cmd.Help() }, + } + + var cmdServiceList = cmdServiceList{common: c.common} + cmd.AddCommand(cmdServiceList.Command()) + + var cmdServiceAdd = cmdServiceAdd{common: c.common} + cmd.AddCommand(cmdServiceAdd.Command()) + + return cmd +} + +type cmdServiceList struct { + common *CmdControl +} + +func (c *cmdServiceList) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List MicroCloud services and their cluster members", + RunE: c.Run, + } + + return cmd +} + +func (c *cmdServiceList) Run(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return cmd.Help() + } + + // Get a microcluster client so we can get state information. + cloudApp, err := microcluster.App(microcluster.Args{StateDir: c.common.FlagMicroCloudDir}) + if err != nil { + return err + } + + // Fetch the name and address, and ensure we're initialized. + status, err := cloudApp.Status(context.Background()) + if err != nil { + return fmt.Errorf("Failed to get MicroCloud status: %w", err) + } + + if !status.Ready { + return fmt.Errorf("MicroCloud is uninitialized, run 'microcloud init' first") + } + + services := []types.ServiceType{types.MicroCloud, types.LXD} + optionalServices := map[types.ServiceType]string{ + types.MicroCeph: api.MicroCephDir, + types.MicroOVN: api.MicroOVNDir, + } + + cfg := initConfig{ + bootstrap: false, + common: c.common, + asker: &c.common.asker, + systems: map[string]InitSystem{}, + state: map[string]service.SystemInformation{}, + } + + cfg.name = status.Name + cfg.address = status.Address.Addr().String() + + services, err = cfg.askMissingServices(services, optionalServices) + if err != nil { + return err + } + + // Instantiate a handler for the services. + s, err := service.NewHandler(status.Name, status.Address.Addr().String(), c.common.FlagMicroCloudDir, c.common.FlagLogDebug, c.common.FlagLogVerbose, services...) + if err != nil { + return err + } + + mu := sync.Mutex{} + header := []string{"NAME", "ADDRESS", "ROLE", "STATUS"} + allClusters := map[types.ServiceType][][]string{} + err = s.RunConcurrent("", "", func(s service.Service) error { + var err error + var data [][]string + var microClient *client.Client + var lxd lxd.InstanceServer + switch s.Type() { + case types.LXD: + lxd, err = s.(*service.LXDService).Client(context.Background(), "") + case types.MicroCeph: + microClient, err = s.(*service.CephService).Client("", "") + case types.MicroOVN: + microClient, err = s.(*service.OVNService).Client() + case types.MicroCloud: + microClient, err = s.(*service.CloudService).Client() + } + + if err != nil { + return err + } + + if microClient != nil { + clusterMembers, err := microClient.GetClusterMembers(context.Background()) + if err != nil && !lxdAPI.StatusErrorCheck(err, http.StatusServiceUnavailable) { + return err + } + + if len(clusterMembers) != 0 { + data = make([][]string, len(clusterMembers)) + for i, clusterMember := range clusterMembers { + data[i] = []string{clusterMember.Name, clusterMember.Address.String(), clusterMember.Role, string(clusterMember.Status)} + } + + sort.Sort(cli.SortColumnsNaturally(data)) + } + } else if lxd != nil { + server, _, err := lxd.GetServer() + if err != nil { + return err + } + + if server.Environment.ServerClustered { + clusterMembers, err := lxd.GetClusterMembers() + if err != nil { + return err + } + + data = make([][]string, len(clusterMembers)) + for i, clusterMember := range clusterMembers { + data[i] = []string{clusterMember.ServerName, clusterMember.URL, strings.Join(clusterMember.Roles, "\n"), string(clusterMember.Status)} + } + + sort.Sort(cli.SortColumnsNaturally(data)) + } + } + + mu.Lock() + allClusters[s.Type()] = data + mu.Unlock() + + return nil + }) + if err != nil { + return err + } + + for serviceType, data := range allClusters { + if len(data) == 0 { + fmt.Printf("%s: Not initialized\n", serviceType) + } else { + fmt.Printf("%s:\n", serviceType) + err = cli.RenderTable(cli.TableFormatTable, header, data, nil) + if err != nil { + return err + } + } + } + + return nil +} + +type cmdServiceAdd struct { + common *CmdControl +} + +func (c *cmdServiceAdd) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "add", + Short: "Add new services to the existing MicroCloud", + RunE: c.Run, + } + + return cmd +} + +func (c *cmdServiceAdd) Run(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return cmd.Help() + } + + cfg := initConfig{ + // Set bootstrap to true because we are setting up a new cluster for new services. + bootstrap: true, + common: c.common, + asker: &c.common.asker, + systems: map[string]InitSystem{}, + state: map[string]service.SystemInformation{}, + } + + // Get a microcluster client so we can get state information. + cloudApp, err := microcluster.App(microcluster.Args{StateDir: c.common.FlagMicroCloudDir}) + if err != nil { + return err + } + + // Fetch the name and address, and ensure we're initialized. + status, err := cloudApp.Status(context.Background()) + if err != nil { + return fmt.Errorf("Failed to get MicroCloud status: %w", err) + } + + if !status.Ready { + return fmt.Errorf("MicroCloud is uninitialized, run 'microcloud init' first") + } + + cfg.name = status.Name + cfg.address = status.Address.Addr().String() + // enable auto setup to skip lookup related questions. + cfg.autoSetup = true + err = cfg.askAddress() + if err != nil { + return err + } + + cfg.autoSetup = false + services := []types.ServiceType{types.MicroCloud, types.LXD} + optionalServices := map[types.ServiceType]string{ + types.MicroCeph: api.MicroCephDir, + types.MicroOVN: api.MicroOVNDir, + } + + // Set the auto flag to true so that we automatically omit any services that aren't installed. + services, err = cfg.askMissingServices(services, optionalServices) + if err != nil { + return err + } + + // Instantiate a handler for the services. + s, err := service.NewHandler(cfg.name, cfg.address, c.common.FlagMicroCloudDir, c.common.FlagLogDebug, c.common.FlagLogVerbose, services...) + if err != nil { + return err + } + + state, err := s.CollectSystemInformation(context.Background(), mdns.ServerInfo{Name: cfg.name, Address: cfg.address, Services: services}) + if err != nil { + return err + } + + cfg.state[cfg.name] = *state + // Create an InitSystem map to carry through the interactive setup. + clusters := cfg.state[cfg.name].ExistingServices + for name, address := range clusters[types.MicroCloud] { + cfg.systems[name] = InitSystem{ + ServerInfo: mdns.ServerInfo{ + Name: name, + Address: address, + Services: services, + }, + } + } + + for _, system := range cfg.systems { + if system.ServerInfo.Name == "" || system.ServerInfo.Name == cfg.name { + continue + } + + state, err := s.CollectSystemInformation(context.Background(), system.ServerInfo) + if err != nil { + return err + } + + cfg.state[system.ServerInfo.Name] = *state + } + + askClusteredServices := []types.ServiceType{} + serviceMap := map[types.ServiceType]bool{} + for _, state := range cfg.state { + localState := cfg.state[s.Name] + if len(state.ExistingServices[types.LXD]) != len(localState.ExistingServices[types.LXD]) || len(state.ExistingServices[types.LXD]) <= 0 { + return fmt.Errorf("Unable to add services. Some systems are not part of the LXD cluster") + } + + if len(state.ExistingServices[types.MicroCeph]) <= 0 && !serviceMap[types.MicroCeph] { + askClusteredServices = append(askClusteredServices, types.MicroCeph) + serviceMap[types.MicroCeph] = true + } + + if len(state.ExistingServices[types.MicroOVN]) <= 0 && !serviceMap[types.MicroOVN] { + askClusteredServices = append(askClusteredServices, types.MicroOVN) + serviceMap[types.MicroOVN] = true + } + } + + if len(askClusteredServices) == 0 { + return fmt.Errorf("All services have already been set up") + } + + err = cfg.askClustered(s, askClusteredServices) + if err != nil { + return err + } + + // Go through the normal setup for disks and networks if necessary. + for _, service := range askClusteredServices { + switch service { + case types.MicroCeph: + err := cfg.askDisks(s) + if err != nil { + return err + } + + case types.MicroOVN: + err := cfg.askNetwork(s) + if err != nil { + return err + } + } + } + + return cfg.setupCluster(s) +} diff --git a/doc/how-to/add_service.md b/doc/how-to/add_service.md new file mode 100644 index 00000000..936cd9c6 --- /dev/null +++ b/doc/how-to/add_service.md @@ -0,0 +1,46 @@ +(howto-add-service)= +# How to add a service + +If you set up the MicroCloud without MicroOVN or MicroCeph initially, you can add those services with the {command}`microcloud service add` command: + + sudo microcloud service add + +If MicroCloud detects a service is installed but not set up, it will ask to configure the service. + +To add MicroCeph: + + ```{note} + To set up distributed storage, you need at least three additional disks on at least three different machines. + The disks must not contain any partitions. + ``` + +1. Select `yes` to set up distributed storage. + + 1. Select the disks that you want to use for distributed storage. + You must select at least three disks. + + 1. Select whether you want to wipe any of the disks. + Wiping a disk will destroy all data on it. + + 1. You can choose to optionally encrypt the chosen disks. + + 1. You can choose to optionally set up a CephFS distributed file system. + + 1. Select either an IPv4 or IPv6 CIDR subnet for the Ceph internal traffic. You can leave it empty to use the default value, which is the MicroCloud internal network (see {ref}`howto-ceph-networking` for how to configure it). + +To add MicroOVN: + +1. Select `yes` to set up distributed networking. + + 1. Select the network interfaces that you want to use (see {ref}`microcloud-networking-uplink`). + You must select one network interface per machine. + + 1. If you want to use IPv4, specify the IPv4 gateway on the uplink network (in CIDR notation) and the first and last IPv4 address in the range that you want to use with LXD. + + 1. If you want to use IPv6, specify the IPv6 gateway on the uplink network (in CIDR notation). + +MicroCloud now starts to bootstrap the cluster for only the new services. + +Monitor the output to see whether all steps complete successfully. + +See {ref}`bootstrapping-process` for more information. diff --git a/doc/how-to/commands.md b/doc/how-to/commands.md index 5cfae791..97caafbc 100644 --- a/doc/how-to/commands.md +++ b/doc/how-to/commands.md @@ -260,7 +260,10 @@ See {ref}`lxd:cluster-manage-instance` and {ref}`lxd:cluster-evacuate`. ```{list-table} :widths: 2 3 - * - Inspect the cluster status + * - Inspect the cluster status for all services at once + - :command:`microcloud service list` + + * - Inspect the cluster status for each service - {command}`microcloud cluster list` {command}`lxc cluster list` diff --git a/doc/how-to/index.md b/doc/how-to/index.md index 77c18d82..b6270e9f 100644 --- a/doc/how-to/index.md +++ b/doc/how-to/index.md @@ -12,6 +12,7 @@ Initialise MicroCloud Configure Ceph networking Add a machine Remove a machine +Add a service Get support Contribute to MicroCloud Work with MicroCloud diff --git a/service/lxd.go b/service/lxd.go index c6029671..baefba03 100644 --- a/service/lxd.go +++ b/service/lxd.go @@ -68,9 +68,16 @@ func (s LXDService) remoteClient(secret string, address string, port int64) (lxd return nil, err } + serverCert, err := s.m.FileSystem.ServerCert() + if err != nil { + return nil, err + } + remoteURL := c.URL() client, err := lxd.ConnectLXD(remoteURL.String(), &lxd.ConnectionArgs{ HTTPClient: c.Client.Client, + TLSClientCert: string(serverCert.PublicKey()), + TLSClientKey: string(serverCert.PrivateKey()), InsecureSkipVerify: true, SkipGetServer: true, Proxy: cloudClient.AuthProxy(secret, types.LXD), diff --git a/service/microcloud.go b/service/microcloud.go index 8e2559a7..87829995 100644 --- a/service/microcloud.go +++ b/service/microcloud.go @@ -76,6 +76,11 @@ func (s *CloudService) StartCloud(ctx context.Context, service *Handler, endpoin }) } +// Client returns a client to the MicroCloud unix socket. +func (s CloudService) Client() (*microClient.Client, error) { + return s.client.LocalClient() +} + // Bootstrap bootstraps the MicroCloud daemon on the default port. func (s CloudService) Bootstrap(ctx context.Context) error { err := s.client.NewCluster(ctx, s.name, util.CanonicalNetworkAddress(s.address, s.port), nil) diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh index 9c283aeb..6f31f391 100644 --- a/test/includes/microcloud.sh +++ b/test/includes/microcloud.sh @@ -2,10 +2,11 @@ # unset_interactive_vars: Unsets all variables related to the test console. unset_interactive_vars() { - unset LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \ + unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \ SETUP_ZFS ZFS_FILTER ZFS_WIPE \ SETUP_CEPH CEPH_WARNING CEPH_FILTER CEPH_WIPE CEPH_ENCRYPT SETUP_CEPHFS CEPH_CLUSTER_NETWORK IGNORE_CEPH_NETWORKING \ - PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET + PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \ + REPLACE_PROFILE } # microcloud_interactive: outputs text that can be passed to `TEST_CONSOLE=1 microcloud init` @@ -13,6 +14,7 @@ unset_interactive_vars() { # The lines that are output are based on the values passed to the listed environment variables. # Any unset variables will be omitted. microcloud_interactive() { + SKIP_LOOKUP=${SKIP_LOOKUP:-} # whether or not to skip the whole lookup block in the interactive command list. LOOKUP_IFACE=${LOOKUP_IFACE:-} # filter string for the lookup interface table. LIMIT_SUBNET=${LIMIT_SUBNET:-} # (yes/no) input for limiting lookup of systems to the above subnet. SKIP_SERVICE=${SKIP_SERVICE:-} # (yes/no) input to skip any missing services. Should be unset if all services are installed. @@ -39,8 +41,11 @@ microcloud_interactive() { IPV4_END=${IPV4_END:-} # OVN ipv4 range end. DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses. IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range. + REPLACE_PROFILE="${REPLACE_PROFILE:-}" # Replace default profile config and devices. - setup=" + setup="" + if ! [ "${SKIP_LOOKUP}" = 1 ]; then + setup=" ${LOOKUP_IFACE} # filter the lookup interface $([ -n "${LOOKUP_IFACE}" ] && printf "select") # select the interface $([ -n "${LOOKUP_IFACE}" ] && printf -- "---") @@ -51,6 +56,7 @@ select-all # select all the sys --- $(true) # workaround for set -e " + fi if [ -n "${REUSE_EXISTING}" ]; then for i in $(seq 1 "${REUSE_EXISTING_COUNT}") ; do @@ -122,6 +128,13 @@ ${IPV6_SUBNET} ${DNS_ADDRESSES} $(true) # workaround for set -e " +fi + +if [ -n "${REPLACE_PROFILE}" ] && [ "${SKIP_LOOKUP}" = 1 ]; then + setup="${setup} +${REPLACE_PROFILE} +$(true) # workaround for set -e +" fi # clear comments and empty lines. @@ -1237,3 +1250,18 @@ ip_config_to_netaddr () { echo "${net_addr}$(ip_prefix_by_netmask "${mask}")" } + +set_cluster_subnet() { + num_systems="${1}" + iface="${2}" + prefix="${3}" + + shift 3 + + for n in $(seq 2 $((num_systems + 1))); do + cluster_ip="${prefix}.${n}/24" + name="$(printf "micro%02d" $((n-1)))" + lxc exec "${name}" -- ip addr flush "${iface}" + lxc exec "${name}" -- ip addr add "${cluster_ip}" dev "${iface}" + done +} diff --git a/test/suites/basic.sh b/test/suites/basic.sh index 0e2ba537..02d85e37 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -1322,3 +1322,133 @@ test_remove_cluster_member() { lxc exec micro01 --env "TEST_CONSOLE=0" -- ${s} cluster list | grep -q "micro03" done } + + +test_add_services() { + unset_interactive_vars + # Set the default config for interactive setup. + + ceph_cluster_subnet_prefix="10.0.1" + ceph_cluster_subnet_iface="enp7s0" + export LOOKUP_IFACE="enp5s0" + export LIMIT_SUBNET="yes" + export EXPECT_PEERS=2 + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="yes" + export SETUP_CEPHFS="yes" + export CEPH_FILTER="lxd_disk2" + export CEPH_WIPE="yes" + export SETUP_OVN="yes" + export OVN_FILTER="enp6s0" + export IPV4_SUBNET="10.1.123.1/24" + export IPV4_START="10.1.123.100" + export IPV4_END="10.1.123.254" + export DNS_ADDRESSES="10.1.123.1,8.8.8.8" + export IPV6_SUBNET="fd42:1:1234:1234::1/64" + export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" + export REPLACE_PROFILE="no" + + reset_systems 3 3 3 + set_cluster_subnet 3 "${ceph_cluster_subnet_iface}" "${ceph_cluster_subnet_prefix}" + echo Add MicroCeph to MicroCloud that was set up without it, and setup remote storage without updating the profile. + lxc exec micro01 -- snap disable microceph + unset SETUP_CEPH + export SKIP_SERVICE="yes" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + lxc exec micro01 -- snap enable microceph + export SETUP_CEPH="yes" + export SKIP_LOOKUP=1 + unset SETUP_ZFS + unset SETUP_OVN + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" + services_validator + + reset_systems 3 3 3 + set_cluster_subnet 3 "${ceph_cluster_subnet_iface}" "${ceph_cluster_subnet_prefix}" + echo Add MicroCeph to MicroCloud that was set up without it, and setup remote storage. + lxc exec micro01 -- snap disable microceph + unset SETUP_CEPH + export SKIP_SERVICE="yes" + export REPLACE_PROFILE="yes" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + lxc exec micro01 -- snap enable microceph + export SETUP_CEPH="yes" + export SKIP_LOOKUP=1 + unset SETUP_ZFS + unset SETUP_OVN + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" + services_validator + + reset_systems 3 3 3 + set_cluster_subnet 3 "${ceph_cluster_subnet_iface}" "${ceph_cluster_subnet_prefix}" + + echo Add MicroOVN to MicroCloud that was set up without it, and setup ovn network + lxc exec micro01 -- snap disable microovn + export SETUP_ZFS="yes" + unset SKIP_LOOKUP + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + lxc exec micro01 -- snap enable microovn + export SETUP_OVN="yes" + export SKIP_LOOKUP=1 + unset SETUP_ZFS + unset SETUP_CEPH + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" + services_validator + + reset_systems 3 3 3 + set_cluster_subnet 3 "${ceph_cluster_subnet_iface}" "${ceph_cluster_subnet_prefix}" + + echo Add both MicroOVN and MicroCeph to a MicroCloud that was set up without it + lxc exec micro01 -- snap disable microovn + lxc exec micro01 -- snap disable microceph + export SETUP_ZFS="yes" + unset SKIP_LOOKUP + unset SETUP_OVN + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + lxc exec micro01 -- snap enable microovn + lxc exec micro01 -- snap enable microceph + export SETUP_OVN="yes" + export SETUP_CEPH="yes" + export SKIP_LOOKUP=1 + unset SETUP_ZFS + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" + services_validator + + reset_systems 3 3 3 + set_cluster_subnet 3 "${ceph_cluster_subnet_iface}" "${ceph_cluster_subnet_prefix}" + + echo Reuse a MicroCeph that was set up on one node of the MicroCloud + lxc exec micro01 -- snap disable microceph + lxc exec micro02 -- microceph cluster bootstrap + export SETUP_ZFS="yes" + unset SETUP_CEPH + unset SKIP_LOOKUP + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + lxc exec micro01 -- snap enable microceph + export REUSE_EXISTING_COUNT=1 + export REUSE_EXISTING="add" + export SETUP_CEPH="yes" + export SKIP_LOOKUP=1 + unset SETUP_ZFS + unset SETUP_OVN + unset CEPH_CLUSTER_NETWORK + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" + services_validator + + reset_systems 3 3 3 + set_cluster_subnet 3 "${ceph_cluster_subnet_iface}" "${ceph_cluster_subnet_prefix}" + + echo Fail to add any services if they have been set up + export SETUP_ZFS="yes" + export SETUP_OVN="yes" + unset REUSE_EXISTING + unset REUSE_EXISTING_COUNT + unset SKIP_LOOKUP + unset SKIP_SERVICE + export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + export SKIP_LOOKUP=1 + ! microcloud_interactive | lxc exec micro01 -- sh -c "microcloud service add > out" || true +}